You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2016/09/26 09:08:19 UTC
[01/50] [abbrv] hadoop git commit: YARN-4232. TopCLI console support
for HA mode. Contributed by Bibin A Chundatt
Repository: hadoop
Updated Branches:
refs/heads/MAPREDUCE-6608 e62053030 -> 14a696f36
YARN-4232. TopCLI console support for HA mode. Contributed by Bibin A Chundatt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ade7c2bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ade7c2bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ade7c2bc
Branch: refs/heads/MAPREDUCE-6608
Commit: ade7c2bc9ccf09d843ccb3dfa56c1453a9e87318
Parents: 501a778
Author: Naganarasimha <na...@apache.org>
Authored: Sat Sep 17 09:52:39 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sat Sep 17 09:52:39 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/cli/TopCLI.java | 114 ++++++++++++++++---
.../hadoop/yarn/client/cli/TestTopCLI.java | 106 +++++++++++++++++
.../hadoop/yarn/webapp/util/WebAppUtils.java | 35 +++---
3 files changed, 224 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade7c2bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
index eca1266..f99dd48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/TopCLI.java
@@ -20,10 +20,13 @@ package org.apache.hadoop.yarn.client.cli;
import java.io.IOException;
import java.io.InputStream;
+import java.net.ConnectException;
+import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLConnection;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.EnumMap;
@@ -37,6 +40,10 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
+import javax.net.ssl.HttpsURLConnection;
+import javax.net.ssl.SSLSocketFactory;
+
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheBuilder;
import org.apache.commons.cli.CommandLine;
@@ -50,7 +57,12 @@ import org.apache.commons.lang.time.DateFormatUtils;
import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpConfig.Policy;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.KerberosAuthenticator;
+import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.ToolRunner;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
@@ -60,12 +72,17 @@ import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueStatistics;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
public class TopCLI extends YarnCLI {
+ private static final String CLUSTER_INFO_URL = "/ws/v1/cluster/info";
+
private static final Log LOG = LogFactory.getLog(TopCLI.class);
private String CLEAR = "\u001b[2J";
private String CLEAR_LINE = "\u001b[2K";
@@ -742,18 +759,12 @@ public class TopCLI extends YarnCLI {
long getRMStartTime() {
try {
- URL url =
- new URL("http://"
- + client.getConfig().get(YarnConfiguration.RM_WEBAPP_ADDRESS)
- + "/ws/v1/cluster/info");
- URLConnection conn = url.openConnection();
- conn.connect();
- InputStream in = conn.getInputStream();
- String encoding = conn.getContentEncoding();
- encoding = encoding == null ? "UTF-8" : encoding;
- String body = IOUtils.toString(in, encoding);
- JSONObject obj = new JSONObject(body);
- JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
+ // connect with url
+ URL url = getClusterUrl();
+ if (null == url) {
+ return -1;
+ }
+ JSONObject clusterInfo = getJSONObject(connect(url));
return clusterInfo.getLong("startedOn");
} catch (Exception e) {
LOG.error("Could not fetch RM start time", e);
@@ -761,6 +772,80 @@ public class TopCLI extends YarnCLI {
return -1;
}
+ private JSONObject getJSONObject(URLConnection conn)
+ throws IOException, JSONException {
+ InputStream in = conn.getInputStream();
+ String encoding = conn.getContentEncoding();
+ encoding = encoding == null ? "UTF-8" : encoding;
+ String body = IOUtils.toString(in, encoding);
+ JSONObject obj = new JSONObject(body);
+ JSONObject clusterInfo = obj.getJSONObject("clusterInfo");
+ return clusterInfo;
+ }
+
+ private URL getClusterUrl() throws Exception {
+ URL url = null;
+ Configuration conf = getConf();
+ if (HAUtil.isHAEnabled(conf)) {
+ Collection<String> haids = HAUtil.getRMHAIds(conf);
+ for (String rmhid : haids) {
+ try {
+ url = getHAClusterUrl(conf, rmhid);
+ if (isActive(url)) {
+ break;
+ }
+ } catch (ConnectException e) {
+ // ignore and try second one when one of RM is down
+ }
+ }
+ } else {
+ url = new URL(
+ WebAppUtils.getRMWebAppURLWithScheme(conf) + CLUSTER_INFO_URL);
+ }
+ return url;
+ }
+
+ private boolean isActive(URL url) throws Exception {
+ URLConnection connect = connect(url);
+ JSONObject clusterInfo = getJSONObject(connect);
+ return clusterInfo.getString("haState").equals("ACTIVE");
+ }
+
+ @VisibleForTesting
+ public URL getHAClusterUrl(Configuration conf, String rmhid)
+ throws MalformedURLException {
+ return new URL(WebAppUtils.getHttpSchemePrefix(conf)
+ + WebAppUtils.getResolvedRemoteRMWebAppURLWithoutScheme(conf,
+ YarnConfiguration.useHttps(conf) ? Policy.HTTPS_ONLY
+ : Policy.HTTP_ONLY,
+ rmhid)
+ + CLUSTER_INFO_URL);
+ }
+
+ private URLConnection connect(URL url) throws Exception {
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ AuthenticatedURL authUrl;
+ SSLFactory clientSslFactory;
+ URLConnection connection;
+ // If https is chosen, configures SSL client.
+ if (YarnConfiguration.useHttps(getConf())) {
+ clientSslFactory = new SSLFactory(SSLFactory.Mode.CLIENT, getConf());
+ clientSslFactory.init();
+ SSLSocketFactory sslSocktFact = clientSslFactory.createSSLSocketFactory();
+
+ authUrl =
+ new AuthenticatedURL(new KerberosAuthenticator(), clientSslFactory);
+ connection = authUrl.openConnection(url, token);
+ HttpsURLConnection httpsConn = (HttpsURLConnection) connection;
+ httpsConn.setSSLSocketFactory(sslSocktFact);
+ } else {
+ authUrl = new AuthenticatedURL(new KerberosAuthenticator());
+ connection = authUrl.openConnection(url, token);
+ }
+ connection.connect();
+ return connection;
+ }
+
String getHeader(QueueMetrics queueMetrics, NodesInformation nodes) {
StringBuilder ret = new StringBuilder();
String queue = "root";
@@ -768,7 +853,10 @@ public class TopCLI extends YarnCLI {
queue = StringUtils.join(queues, ",");
}
long now = Time.now();
- long uptime = now - rmStartTime;
+ long uptime = 0L;
+ if (rmStartTime != -1) {
+ uptime = now - rmStartTime;
+ }
long days = TimeUnit.MILLISECONDS.toDays(uptime);
long hours =
TimeUnit.MILLISECONDS.toHours(uptime)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade7c2bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java
new file mode 100644
index 0000000..706400f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestTopCLI.java
@@ -0,0 +1,106 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Test class for TopCli.
+ *
+ */
+public class TestTopCLI {
+
+ private static final String RM1_NODE_ID = "rm1";
+ private static final String RM2_NODE_ID = "rm2";
+
+ private static List<String> dummyHostNames =
+ Arrays.asList("host1", "host2", "host3");
+
+ private static Map<String, String> savedStaticResolution = new HashMap<>();
+
+ @BeforeClass
+ public static void initializeDummyHostnameResolution() throws Exception {
+ String previousIpAddress;
+ for (String hostName : dummyHostNames) {
+ previousIpAddress = NetUtils.getStaticResolution(hostName);
+ if (null != previousIpAddress) {
+ savedStaticResolution.put(hostName, previousIpAddress);
+ }
+ NetUtils.addStaticResolution(hostName, "10.20.30.40");
+ }
+ }
+
+ @AfterClass
+ public static void restoreDummyHostnameResolution() throws Exception {
+ for (Map.Entry<String, String> hostnameToIpEntry : savedStaticResolution
+ .entrySet()) {
+ NetUtils.addStaticResolution(hostnameToIpEntry.getKey(),
+ hostnameToIpEntry.getValue());
+ }
+ }
+
+ @Test
+ public void testHAClusterInfoURL() throws IOException, InterruptedException {
+ TopCLI topcli = new TopCLI();
+ // http
+ String rm1Address = "host2:8088";
+ String rm2Address = "host3:8088";
+ Configuration conf = topcli.getConf();
+ conf.set(YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM1_NODE_ID,
+ rm1Address);
+ topcli.getConf().set(
+ YarnConfiguration.RM_WEBAPP_ADDRESS + "." + RM2_NODE_ID, rm2Address);
+ topcli.getConf().setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ topcli.getConf().set(YarnConfiguration.RM_HA_IDS,
+ RM1_NODE_ID + "," + RM2_NODE_ID);
+ URL clusterUrl = topcli.getHAClusterUrl(conf, RM1_NODE_ID);
+ Assert.assertEquals("http", clusterUrl.getProtocol());
+ Assert.assertEquals(rm1Address, clusterUrl.getAuthority());
+ clusterUrl = topcli.getHAClusterUrl(conf, RM2_NODE_ID);
+ Assert.assertEquals("http", clusterUrl.getProtocol());
+ Assert.assertEquals(rm2Address, clusterUrl.getAuthority());
+ // https
+ rm1Address = "host2:9088";
+ rm2Address = "host3:9088";
+ conf = topcli.getConf();
+ conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + RM1_NODE_ID,
+ rm1Address);
+ conf.set(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS + "." + RM2_NODE_ID,
+ rm2Address);
+ conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, RM1_NODE_ID + "," + RM2_NODE_ID);
+ conf.set(YarnConfiguration.YARN_HTTP_POLICY_KEY, "HTTPS_ONLY");
+ clusterUrl = topcli.getHAClusterUrl(conf, RM1_NODE_ID);
+ Assert.assertEquals("https", clusterUrl.getProtocol());
+ Assert.assertEquals(rm1Address, clusterUrl.getAuthority());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ade7c2bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 0139500..89f0551 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.RMHAUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -183,32 +182,32 @@ public class WebAppUtils {
public static String getResolvedRemoteRMWebAppURLWithoutScheme(Configuration conf,
Policy httpPolicy) {
- InetSocketAddress address = null;
String rmId = null;
if (HAUtil.isHAEnabled(conf)) {
// If HA enabled, pick one of the RM-IDs and rely on redirect to go to
// the Active RM
rmId = (String) HAUtil.getRMHAIds(conf).toArray()[0];
}
+ return getResolvedRemoteRMWebAppURLWithoutScheme(conf, httpPolicy, rmId);
+ }
+
+ public static String getResolvedRemoteRMWebAppURLWithoutScheme(
+ Configuration conf, Policy httpPolicy, String rmId) {
+ InetSocketAddress address = null;
if (httpPolicy == Policy.HTTPS_ONLY) {
- address =
- conf.getSocketAddr(
- rmId == null
- ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
- : HAUtil.addSuffix(
- YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS, rmId),
- YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
+ address = conf.getSocketAddr(
+ rmId == null ? YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS
+ : HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+ rmId),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT);
} else {
- address =
- conf.getSocketAddr(
- rmId == null
- ? YarnConfiguration.RM_WEBAPP_ADDRESS
- : HAUtil.addSuffix(
- YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
- YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
- YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+ address = conf.getSocketAddr(
+ rmId == null ? YarnConfiguration.RM_WEBAPP_ADDRESS
+ : HAUtil.addSuffix(YarnConfiguration.RM_WEBAPP_ADDRESS, rmId),
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
}
return getResolvedAddress(address);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HDFS-10876. Dispatcher#dispatch
should log IOException stacktrace. Contributed by Manoj Govindassamy.
Posted by ji...@apache.org.
HDFS-10876. Dispatcher#dispatch should log IOException stacktrace. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74b3dd51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74b3dd51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74b3dd51
Branch: refs/heads/MAPREDUCE-6608
Commit: 74b3dd514c86b46197e2e19d9824a423715cab30
Parents: 6e849cb
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Fri Sep 23 13:26:57 2016 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Fri Sep 23 13:26:57 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74b3dd51/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index a75cb30..aea0ae4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -354,7 +354,7 @@ public class Dispatcher {
target.getDDatanode().setHasSuccess();
LOG.info("Successfully moved " + this);
} catch (IOException e) {
- LOG.warn("Failed to move " + this + ": " + e.getMessage());
+ LOG.warn("Failed to move " + this, e);
target.getDDatanode().setHasFailure();
// Proxy or target may have some issues, delay before using these nodes
// further in order to avoid a potential storm of "threads quota
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: MAPREDUCE-6632.
Master.getMasterAddress() should be updated to use YARN-4629 (templedf via
rkanter)
Posted by ji...@apache.org.
MAPREDUCE-6632. Master.getMasterAddress() should be updated to use YARN-4629 (templedf via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4fc632ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4fc632ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4fc632ae
Branch: refs/heads/MAPREDUCE-6608
Commit: 4fc632ae19a1d6b0ec09cc7ead789a3cab1c2f1c
Parents: 40acace
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Sep 22 16:12:56 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Sep 22 16:12:56 2016 -0700
----------------------------------------------------------------------
.../hadoop-mapreduce-client-core/pom.xml | 6 +-
.../java/org/apache/hadoop/mapred/Master.java | 70 ++++++--------------
.../org/apache/hadoop/mapred/TestMaster.java | 56 +---------------
3 files changed, 28 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 4de24c7..00bb11b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -37,7 +37,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-common</artifactId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
index d84e395..ec00405 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Master.java
@@ -19,75 +19,45 @@
package org.apache.hadoop.mapred;
import java.io.IOException;
-import java.net.InetSocketAddress;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.yarn.conf.HAUtil;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.client.util.YarnClientUtils;
@Private
@Unstable
public class Master {
-
- private static final Log LOG = LogFactory.getLog(Master.class);
-
public enum State {
INITIALIZING, RUNNING;
}
- public static String getMasterUserName(Configuration conf) {
- String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
- return conf.get(MRConfig.MASTER_USER_NAME);
- }
- else {
- return conf.get(YarnConfiguration.RM_PRINCIPAL);
- }
+ public static String getMasterAddress(Configuration conf) {
+ String masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
+
+ return NetUtils.createSocketAddr(masterAddress, 8012,
+ MRConfig.MASTER_ADDRESS).getHostName();
}
-
- public static InetSocketAddress getMasterAddress(Configuration conf) {
- String masterAddress;
- String framework = conf.get(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
+
+ public static String getMasterPrincipal(Configuration conf)
+ throws IOException {
+ String masterPrincipal;
+ String framework = conf.get(MRConfig.FRAMEWORK_NAME,
+ MRConfig.YARN_FRAMEWORK_NAME);
+
if (framework.equals(MRConfig.CLASSIC_FRAMEWORK_NAME)) {
- masterAddress = conf.get(MRConfig.MASTER_ADDRESS, "localhost:8012");
- return NetUtils.createSocketAddr(masterAddress, 8012, MRConfig.MASTER_ADDRESS);
- } else if (framework.equals(MRConfig.YARN_FRAMEWORK_NAME) &&
- HAUtil.isHAEnabled(conf)) {
- YarnConfiguration yarnConf = new YarnConfiguration(conf);
- if (yarnConf.get(YarnConfiguration.RM_HA_ID) == null) {
- String[] rmIds = yarnConf.getStrings(YarnConfiguration.RM_HA_IDS);
- if (rmIds != null && rmIds.length > 0) {
- // If RM_HA_ID is not configured, use the first one.
- // Because any valid RM HA ID should work.
- yarnConf.set(YarnConfiguration.RM_HA_ID, rmIds[0]);
- } else {
- LOG.warn("RM_HA_IDS is not configured when RM HA is enabled");
- }
- }
- return yarnConf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ String masterAddress = getMasterAddress(conf);
+ // get kerberos principal for use as delegation token renewer
+ masterPrincipal =
+ SecurityUtil.getServerPrincipal(conf.get(MRConfig.MASTER_USER_NAME),
+ masterAddress);
} else {
- return conf.getSocketAddr(
- YarnConfiguration.RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_ADDRESS,
- YarnConfiguration.DEFAULT_RM_PORT);
+ masterPrincipal = YarnClientUtils.getRmPrincipal(conf);
}
- }
- public static String getMasterPrincipal(Configuration conf)
- throws IOException {
- String masterHostname = getMasterAddress(conf).getHostName();
- // get kerberos principal for use as delegation token renewer
- return SecurityUtil.getServerPrincipal(getMasterUserName(conf), masterHostname);
+ return masterPrincipal;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4fc632ae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
index 498abbc..0f4ebee 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestMaster.java
@@ -18,13 +18,10 @@
package org.apache.hadoop.mapred;
-import static org.junit.Assert.*;
-
-import java.net.InetSocketAddress;
-
import org.apache.hadoop.mapreduce.MRConfig;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import org.junit.Test;
public class TestMaster {
@@ -33,13 +30,6 @@ public class TestMaster {
public void testGetMasterAddress() {
YarnConfiguration conf = new YarnConfiguration();
- // Default is yarn framework
- String masterHostname = Master.getMasterAddress(conf).getHostName();
-
- // no address set so should default to default rm address
- InetSocketAddress rmAddr = NetUtils.createSocketAddr(YarnConfiguration.DEFAULT_RM_ADDRESS);
- assertEquals(masterHostname, rmAddr.getHostName());
-
// Trying invalid master address for classic
conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
conf.set(MRConfig.MASTER_ADDRESS, "local:invalid");
@@ -55,47 +45,7 @@ public class TestMaster {
// Change master address to a valid value
conf.set(MRConfig.MASTER_ADDRESS, "bar.com:8042");
- masterHostname = Master.getMasterAddress(conf).getHostName();
+ String masterHostname = Master.getMasterAddress(conf);
assertEquals(masterHostname, "bar.com");
-
- // change framework to yarn
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- conf.set(YarnConfiguration.RM_ADDRESS, "foo1.com:8192");
- masterHostname = Master.getMasterAddress(conf).getHostName();
- assertEquals(masterHostname, "foo1.com");
-
- // change framework to yarn and enable HA
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
- conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
- conf.set(YarnConfiguration.RM_ADDRESS + ".rm1", "rm1.com:8192");
- conf.set(YarnConfiguration.RM_ADDRESS + ".rm2", "rm2.com:8192");
- masterHostname = Master.getMasterAddress(conf).getHostName();
- // If RM_HA_ID is not configured, the first one in RM_HA_IDS will be used.
- assertEquals(masterHostname, "rm1.com");
- conf.set(YarnConfiguration.RM_HA_ID, "rm2");
- masterHostname = Master.getMasterAddress(conf).getHostName();
- // If RM_HA_ID is configured, use the given RM_HA_ID.
- assertEquals(masterHostname, "rm2.com");
}
-
- @Test
- public void testGetMasterUser() {
- YarnConfiguration conf = new YarnConfiguration();
- conf.set(MRConfig.MASTER_USER_NAME, "foo");
- conf.set(YarnConfiguration.RM_PRINCIPAL, "bar");
-
- // default is yarn framework
- assertEquals(Master.getMasterUserName(conf), "bar");
-
- // set framework name to classic
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
- assertEquals(Master.getMasterUserName(conf), "foo");
-
- // change framework to yarn
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.YARN_FRAMEWORK_NAME);
- assertEquals(Master.getMasterUserName(conf), "bar");
-
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: HDFS-10489. Deprecate
dfs.encryption.key.provider.uri for HDFS encryption zones. Contributed by
Xiao Chen.
Posted by ji...@apache.org.
HDFS-10489. Deprecate dfs.encryption.key.provider.uri for HDFS encryption zones. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea839bd4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea839bd4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea839bd4
Branch: refs/heads/MAPREDUCE-6608
Commit: ea839bd48e4478fc7b6d0a69e0eaeae2de5e0f0d
Parents: 96142ef
Author: Xiao Chen <xi...@apache.org>
Authored: Sat Sep 17 22:25:39 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Sat Sep 17 22:25:39 2016 -0700
----------------------------------------------------------------------
.../hadoop/crypto/key/KeyProviderFactory.java | 3 ++-
.../hadoop/fs/CommonConfigurationKeysPublic.java | 8 ++++++++
.../src/main/resources/core-default.xml | 8 ++++++++
.../src/site/markdown/DeprecatedProperties.md | 1 +
.../hadoop-kms/src/site/markdown/index.md.vm | 10 +++++-----
.../java/org/apache/hadoop/hdfs/DFSUtilClient.java | 13 +++++++------
.../org/apache/hadoop/hdfs/HdfsConfiguration.java | 3 +++
.../org/apache/hadoop/hdfs/KeyProviderCache.java | 6 +++---
.../hadoop/hdfs/client/HdfsClientConfigKeys.java | 1 -
.../org/apache/hadoop/test/TestHdfsHelper.java | 4 +++-
.../hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java | 4 ++--
.../java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 --
.../src/main/resources/hdfs-default.xml | 8 --------
.../src/site/markdown/TransparentEncryption.md | 2 +-
.../org/apache/hadoop/cli/TestCryptoAdminCLI.java | 4 ++--
.../org/apache/hadoop/hdfs/TestAclsEndToEnd.java | 3 ++-
.../java/org/apache/hadoop/hdfs/TestDFSUtil.java | 12 ++++++++----
.../apache/hadoop/hdfs/TestEncryptionZones.java | 17 ++++++++++-------
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 3 ++-
.../apache/hadoop/hdfs/TestKeyProviderCache.java | 10 +++++-----
.../apache/hadoop/hdfs/TestReservedRawPaths.java | 3 ++-
.../hdfs/TestSecureEncryptionZoneWithKMS.java | 6 ++++--
.../server/namenode/TestNestedEncryptionZones.java | 4 +++-
.../namenode/metrics/TestNameNodeMetrics.java | 3 ++-
24 files changed, 83 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
index ce99d79..b16960c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderFactory.java
@@ -29,6 +29,7 @@ import java.util.ServiceLoader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
* A factory to create a list of KeyProvider based on the path given in a
@@ -39,7 +40,7 @@ import org.apache.hadoop.conf.Configuration;
@InterfaceStability.Unstable
public abstract class KeyProviderFactory {
public static final String KEY_PROVIDER_PATH =
- "hadoop.security.key.provider.path";
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
public abstract KeyProvider createProvider(URI providerName,
Configuration conf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
index 0a3afb7..b5b107c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/CommonConfigurationKeysPublic.java
@@ -628,6 +628,14 @@ public class CommonConfigurationKeysPublic {
public static final String HADOOP_SECURITY_IMPERSONATION_PROVIDER_CLASS =
"hadoop.security.impersonation.provider.class";
+ /**
+ * @see
+ * <a href="{@docRoot}/../hadoop-project-dist/hadoop-common/core-default.xml">
+ * core-default.xml</a>
+ */
+ public static final String HADOOP_SECURITY_KEY_PROVIDER_PATH =
+ "hadoop.security.key.provider.path";
+
// <!-- KMSClientProvider configurations -->
/**
* @see
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index fda8c9b..5b8d49d 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2038,6 +2038,14 @@
</property>
<property>
+ <name>hadoop.security.key.provider.path</name>
+ <description>
+ The KeyProvider to use when managing zone keys, and interacting with
+ encryption keys when reading and writing to an encryption zone.
+ </description>
+</property>
+
+<property>
<name>fs.har.impl.disable.cache</name>
<value>true</value>
<description>Don't cache 'har' filesystem instances.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
index adebe6e..94ed68e 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -28,6 +28,7 @@ The following table lists the configuration property names that are deprecated i
| dfs.data.dir | dfs.datanode.data.dir |
| dfs.datanode.max.xcievers | dfs.datanode.max.transfer.threads |
| dfs.df.interval | fs.df.interval |
+| dfs.encryption.key.provider.uri | hadoop.security.key.provider.path |
| dfs.federation.nameservice.id | dfs.nameservice.id |
| dfs.federation.nameservices | dfs.nameservices |
| dfs.http.address | dfs.namenode.http-address |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 889dbaf1..0c6d0b2 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -37,10 +37,10 @@ KMS Client Configuration
The KMS client `KeyProvider` uses the **kms** scheme, and the embedded URL must be the URL of the KMS. For example, for a KMS running on `http://localhost:9600/kms`, the KeyProvider URI is `kms://http@localhost:9600/kms`. And, for a KMS running on `https://localhost:9600/kms`, the KeyProvider URI is `kms://https@localhost:9600/kms`
The following is an example to configure HDFS NameNode as a KMS client in
-`hdfs-site.xml`:
+`core-site.xml`:
<property>
- <name>dfs.encryption.key.provider.uri</name>
+ <name>hadoop.security.key.provider.path</name>
<value>kms://http@localhost:9600/kms</value>
<description>
The KeyProvider to use when interacting with encryption keys used
@@ -664,15 +664,15 @@ is to use LoadBalancingKMSClientProvider. Using this approach, a KMS client
(for example, a HDFS NameNode) is aware of multiple KMS instances, and it sends
requests to them in a round-robin fashion. LoadBalancingKMSClientProvider is
implicitly used when more than one URI is specified in
-`dfs.encryption.key.provider.uri`.
+`hadoop.security.key.provider.path`.
-The following example in `hdfs-site.xml` configures two KMS
+The following example in `core-site.xml` configures two KMS
instances, `kms01.example.com` and `kms02.example.com`.
The hostnames are separated by semi-colons, and all KMS instances must run
on the same port.
<property>
- <name>dfs.encryption.key.provider.uri</name>
+ <name>hadoop.security.key.provider.path</name>
<value>kms://https@kms01.example.com;kms02.example.com:9600/kms</value>
<description>
The KeyProvider to use when interacting with encryption keys used
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
index c511596..b93632a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSUtilClient.java
@@ -526,7 +526,7 @@ public class DFSUtilClient {
}
private static String keyProviderUriKeyName =
- HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI;
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH;
/**
* Set the key provider uri configuration key name for creating key providers.
@@ -616,16 +616,17 @@ public class DFSUtilClient {
}
/**
- * Probe for HDFS Encryption being enabled; this uses the value of
- * the option {@link HdfsClientConfigKeys#DFS_ENCRYPTION_KEY_PROVIDER_URI},
- * returning true if that property contains a non-empty, non-whitespace
+ * Probe for HDFS Encryption being enabled; this uses the value of the option
+ * {@link CommonConfigurationKeysPublic#HADOOP_SECURITY_KEY_PROVIDER_PATH}
+ * , returning true if that property contains a non-empty, non-whitespace
* string.
* @param conf configuration to probe
* @return true if encryption is considered enabled.
*/
public static boolean isHDFSEncryptionEnabled(Configuration conf) {
- return !conf.getTrimmed(
- HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "").isEmpty();
+ return !(conf.getTrimmed(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "")
+ .isEmpty());
}
public static InetSocketAddress getNNAddress(String address) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
index 580e7f9..3f6eae0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/HdfsConfiguration.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.DeprecatedKeys;
@@ -141,6 +142,8 @@ public class HdfsConfiguration extends Configuration {
HdfsClientConfigKeys.DFS_NAMESERVICES),
new DeprecationDelta("dfs.federation.nameservice.id",
DeprecatedKeys.DFS_NAMESERVICE_ID),
+ new DeprecationDelta("dfs.encryption.key.provider.uri",
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH),
});
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
index 0b0e006..3491010 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
@@ -25,7 +25,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.cache.Cache;
@@ -86,11 +86,11 @@ public class KeyProviderCache {
private URI createKeyProviderURI(Configuration conf) {
final String providerUriStr = conf.getTrimmed(
- HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "");
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH, "");
// No provider set in conf
if (providerUriStr.isEmpty()) {
LOG.error("Could not find uri with key ["
- + HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI
+ + CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH
+ "] to create a keyProvider !!");
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 3ea42c3..642d4c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -139,7 +139,6 @@ public interface HdfsClientConfigKeys {
"dfs.datanode.kerberos.principal";
String DFS_DATANODE_READAHEAD_BYTES_KEY = "dfs.datanode.readahead.bytes";
long DFS_DATANODE_READAHEAD_BYTES_DEFAULT = 4 * 1024 * 1024; // 4MB
- String DFS_ENCRYPTION_KEY_PROVIDER_URI = "dfs.encryption.key.provider.uri";
String DFS_ENCRYPT_DATA_TRANSFER_CIPHER_SUITES_KEY =
"dfs.encrypt.data.transfer.cipher.suites";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
index 5695285..0e701f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/test/TestHdfsHelper.java
@@ -22,6 +22,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
@@ -158,7 +159,8 @@ public class TestHdfsHelper extends TestDirHelper {
FileSystemTestHelper helper = new FileSystemTestHelper();
final String jceksPath = JavaKeyStoreProvider.SCHEME_NAME + "://file" +
new Path(helper.getTestRootDir(), "test.jks").toUri();
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, jceksPath);
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ jceksPath);
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
builder.numDataNodes(2);
MiniDFSCluster miniHdfs = builder.build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
index 811aeda..0fabb30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/src/test/java/org/apache/hadoop/hdfs/nfs/nfs3/TestRpcProgramNfs3.java
@@ -30,10 +30,10 @@ import java.util.EnumSet;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
@@ -135,7 +135,7 @@ public class TestRpcProgramNfs3 {
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
- config.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ config.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
ProxyUsers.refreshSuperUserGroupsConfiguration(config);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index caf6b60..3532d25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -804,8 +804,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
HdfsClientConfigKeys.DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY;
public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100;
public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses";
- public static final String DFS_ENCRYPTION_KEY_PROVIDER_URI =
- HdfsClientConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI;
public static final String DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_KEY = "dfs.namenode.edekcacheloader.interval.ms";
public static final int DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_DEFAULT = 1000;
public static final String DFS_NAMENODE_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY = "dfs.namenode.edekcacheloader.initial.delay.ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 29c9ef2..ebaefde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2757,14 +2757,6 @@
</property>
<property>
- <name>dfs.encryption.key.provider.uri</name>
- <description>
- The KeyProvider to use when interacting with encryption keys used
- when reading and writing to an encryption zone.
- </description>
-</property>
-
-<property>
<name>dfs.storage.policy.enabled</name>
<value>true</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index 6d0e52c..ee98df8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -117,7 +117,7 @@ Once a KMS has been set up and the NameNode and HDFS clients have been correctly
### <a name="Configuring_the_cluster_KeyProvider"></a>Configuring the cluster KeyProvider
-#### dfs.encryption.key.provider.uri
+#### hadoop.security.key.provider.path
The KeyProvider to use when interacting with encryption keys used when reading and writing to an encryption zone.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
index 7c6aaeb..99a7c2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestCryptoAdminCLI.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.cli.util.CommandExecutor.Result;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
@@ -66,7 +66,7 @@ public class TestCryptoAdminCLI extends CLITestHelperDFS {
tmpDir = GenericTestUtils.getTestDir(UUID.randomUUID().toString());
final Path jksPath = new Path(tmpDir.toString(), "test.jks");
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri());
dfsCluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
index 2b515d0..3ff7055 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestAclsEndToEnd.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration;
import org.apache.hadoop.crypto.key.kms.server.KeyAuthorizationKeyProvider;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystemTestHelper;
@@ -190,7 +191,7 @@ public class TestAclsEndToEnd {
"keyadmin,hdfs,user");
conf.set(ProxyUsers.CONF_HADOOP_PROXYUSER + "." + realUser + ".hosts",
"*");
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
getKeyProviderURI());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index b7db601..7257bbd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -1031,16 +1032,19 @@ public class TestDFSUtil {
@Test
public void testEncryptionProbe() throws Throwable {
Configuration conf = new Configuration(false);
- conf.unset(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI);
+ conf.unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
assertFalse("encryption enabled on no provider key",
DFSUtilClient.isHDFSEncryptionEnabled(conf));
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "");
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ "");
assertFalse("encryption enabled on empty provider key",
DFSUtilClient.isHDFSEncryptionEnabled(conf));
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "\n\t\n");
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ "\n\t\n");
assertFalse("encryption enabled on whitespace provider key",
DFSUtilClient.isHDFSEncryptionEnabled(conf));
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "http://hadoop.apache.org");
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ "http://hadoop.apache.org");
assertTrue("encryption disabled on valid provider key",
DFSUtilClient.isHDFSEncryptionEnabled(conf));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 53c12ec..b634dd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -150,7 +150,8 @@ public class TestEncryptionZones {
// Set up java key store
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, getKeyProviderURI());
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ getKeyProviderURI());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
// Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
@@ -845,9 +846,9 @@ public class TestEncryptionZones {
// Check KeyProvider state
// Flushing the KP on the NN, since it caches, and init a test one
cluster.getNamesystem().getProvider().flush();
- KeyProvider provider = KeyProviderFactory
- .get(new URI(conf.getTrimmed(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI)),
- conf);
+ KeyProvider provider = KeyProviderFactory.get(new URI(conf.getTrimmed(
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH)),
+ conf);
List<String> keys = provider.getKeys();
assertEquals("Expected NN to have created one key per zone", 1,
keys.size());
@@ -931,7 +932,8 @@ public class TestEncryptionZones {
public void testCreateEZWithNoProvider() throws Exception {
// Unset the key provider and make sure EZ ops don't work
final Configuration clusterConf = cluster.getConfiguration(0);
- clusterConf.unset(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI);
+ clusterConf
+ .unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
cluster.restartNameNode(true);
cluster.waitActive();
final Path zone1 = new Path("/zone1");
@@ -943,8 +945,9 @@ public class TestEncryptionZones {
assertExceptionContains("since no key provider is available", e);
}
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
- clusterConf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
- JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
+ clusterConf
+ .set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
);
// Try listing EZs as well
assertNumZones(0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
index f3a2640..bb8b79b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
@@ -62,7 +63,7 @@ public class TestEncryptionZonesWithHA {
fsHelper = new FileSystemTestHelper();
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
JavaKeyStoreProvider.SCHEME_NAME + "://file" +
new Path(testRootDir.toString(), "test.jks").toUri()
);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
index 4cbe871..0868b54 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
@@ -24,7 +24,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
-import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.junit.Assert;
import org.junit.Test;
@@ -94,26 +94,26 @@ public class TestKeyProviderCache {
public void testCache() throws Exception {
KeyProviderCache kpCache = new KeyProviderCache(10000);
Configuration conf = new Configuration();
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
"dummy://foo:bar@test_provider1");
KeyProvider keyProvider1 = kpCache.get(conf);
Assert.assertNotNull("Returned Key Provider is null !!", keyProvider1);
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
"dummy://foo:bar@test_provider1");
KeyProvider keyProvider2 = kpCache.get(conf);
Assert.assertTrue("Different KeyProviders returned !!",
keyProvider1 == keyProvider2);
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
"dummy://test_provider3");
KeyProvider keyProvider3 = kpCache.get(conf);
Assert.assertFalse("Same KeyProviders returned !!",
keyProvider1 == keyProvider3);
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
"dummy://hello:there@test_provider1");
KeyProvider keyProvider4 = kpCache.get(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index c09d346..5416739 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -25,6 +25,7 @@ import java.util.EnumSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileContextTestWrapper;
import org.apache.hadoop.fs.FileStatus;
@@ -77,7 +78,7 @@ public class TestReservedRawPaths {
String testRoot = fsHelper.getTestRootDir();
File testRootDir = new File(testRoot).getAbsoluteFile();
final Path jksPath = new Path(testRootDir.toString(), "test.jks");
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
JavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri()
);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
index b85c069..96e39ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSecureEncryptionZoneWithKMS.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
import org.apache.hadoop.crypto.key.kms.server.KMSConfiguration;
import org.apache.hadoop.crypto.key.kms.server.MiniKMS;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestWrapper;
import org.apache.hadoop.fs.FileUtil;
@@ -237,8 +238,9 @@ public class TestSecureEncryptionZoneWithKMS {
@Before
public void setup() throws Exception {
// Start MiniDFS Cluster
- baseConf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
- getKeyProviderURI());
+ baseConf
+ .set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ getKeyProviderURI());
baseConf.setBoolean(DFSConfigKeys
.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java
index 8847c91..13fc985 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNestedEncryptionZones.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -85,7 +86,8 @@ public class TestNestedEncryptionZones {
// Set up java key store
String testRoot = fsHelper.getTestRootDir();
testRootDir = new File(testRoot).getAbsoluteFile();
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, getKeyProviderURI());
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ getKeyProviderURI());
conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
// Lower the batch size for testing
conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea839bd4/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index 9e95650..90d61ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.server.namenode.metrics;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FileSystemTestWrapper;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -642,7 +643,7 @@ public class TestNameNodeMetrics {
// Set up java key store
String testRoot = fsHelper.getTestRootDir();
File testRootDir = new File(testRoot).getAbsoluteFile();
- conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
JavaKeyStoreProvider.SCHEME_NAME + "://file" +
new Path(testRootDir.toString(), "test.jks").toUri());
conf.setBoolean(DFSConfigKeys
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-5655.
TestContainerManagerSecurity#testNMTokens is asserting. Contributed by Robert
Kanter
Posted by ji...@apache.org.
YARN-5655. TestContainerManagerSecurity#testNMTokens is asserting. Contributed by Robert Kanter
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6d1d742
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6d1d742
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6d1d742
Branch: refs/heads/MAPREDUCE-6608
Commit: c6d1d742e70e7b8f1d89cf9a4780657646e6a367
Parents: 734d54c
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Sep 20 14:15:06 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Sep 20 14:15:06 2016 +0000
----------------------------------------------------------------------
.../apache/hadoop/yarn/server/TestContainerManagerSecurity.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6d1d742/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
index ee3396d..408c1cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
@@ -68,6 +68,8 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.security.BaseNMTokenSecretManager;
@@ -205,6 +207,9 @@ public class TestContainerManagerSecurity extends KerberosSecurityTestcase {
Resource r = Resource.newInstance(1024, 1);
ApplicationId appId = ApplicationId.newInstance(1, 1);
+ MockRMApp m = new MockRMApp(appId.getId(), appId.getClusterTimestamp(),
+ RMAppState.NEW);
+ yarnCluster.getResourceManager().getRMContext().getRMApps().put(appId, m);
ApplicationAttemptId validAppAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-5622.
TestYarnCLI.testGetContainers fails due to mismatched date formats.
Contributed by Eric Badger.
Posted by ji...@apache.org.
YARN-5622. TestYarnCLI.testGetContainers fails due to mismatched date formats. Contributed by Eric Badger.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6e849cb6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6e849cb6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6e849cb6
Branch: refs/heads/MAPREDUCE-6608
Commit: 6e849cb658438c0561de485e01f3de7df47bf9ad
Parents: a5bb88c
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 23 14:09:25 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 23 14:09:25 2016 -0500
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/client/cli/TestYarnCLI.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6e849cb6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 3fdea40..ef64365 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -35,8 +35,6 @@ import java.io.OutputStreamWriter;
import java.io.PrintStream;
import java.io.PrintWriter;
import java.io.UnsupportedEncodingException;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
@@ -81,6 +79,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.Times;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -300,7 +299,6 @@ public class TestYarnCLI {
reports.add(container);
reports.add(container1);
reports.add(container2);
- DateFormat dateFormat=new SimpleDateFormat("EEE MMM dd HH:mm:ss Z yyyy");
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
sysOutStream.reset();
@@ -316,13 +314,13 @@ public class TestYarnCLI {
pw.printf(ApplicationCLI.CONTAINER_PATTERN, "Container-Id", "Start Time",
"Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
pw.printf(ApplicationCLI.CONTAINER_PATTERN, "container_1234_0005_01_000001",
- dateFormat.format(new Date(time1)), dateFormat.format(new Date(time2)),
+ Times.format(time1), Times.format(time2),
"COMPLETE", "host:1234", "http://host:2345", "logURL");
pw.printf(ApplicationCLI.CONTAINER_PATTERN, "container_1234_0005_01_000002",
- dateFormat.format(new Date(time1)), dateFormat.format(new Date(time2)),
+ Times.format(time1), Times.format(time2),
"COMPLETE", "host:1234", "http://host:2345", "logURL");
pw.printf(ApplicationCLI.CONTAINER_PATTERN, "container_1234_0005_01_000003",
- dateFormat.format(new Date(time1)), "N/A", "RUNNING", "host:1234",
+ Times.format(time1), "N/A", "RUNNING", "host:1234",
"http://host:2345", "");
pw.close();
String appReportStr = baos.toString("UTF-8");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: Addendum patch for fix javadocs
failure which is caused by YARN-3141. (wangda)
Posted by ji...@apache.org.
Addendum patch for fix javadocs failure which is caused by YARN-3141. (wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e45307c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e45307c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e45307c9
Branch: refs/heads/MAPREDUCE-6608
Commit: e45307c9a063248fcfb08281025d87c4abd343b1
Parents: c6d1d74
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Sep 20 11:21:01 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 20 11:21:01 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e45307c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index f40ecd7..fd43e74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -328,7 +328,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
* of the resources that will be allocated to and preempted from this
* application.
*
- * @param rc
+ * @param resourceCalculator
* @param clusterResource
* @param minimumAllocation
* @return an allocation
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-5539. TimelineClient failed
to retry on java.net.SocketTimeoutException: Read timed out (Junping Du via
Varun Saxena)
Posted by ji...@apache.org.
YARN-5539. TimelineClient failed to retry on java.net.SocketTimeoutException: Read timed out (Junping Du via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a2d7b8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a2d7b8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a2d7b8
Branch: refs/heads/MAPREDUCE-6608
Commit: b8a2d7b8fc96302ba1ef99d24392f463734f1b82
Parents: 3293a7d
Author: Varun Saxena <va...@apache.org>
Authored: Fri Sep 23 13:27:31 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Sep 23 13:27:31 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a2d7b8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 56da2a7..dc4d3e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -265,7 +265,8 @@ public class TimelineClientImpl extends TimelineClient {
public boolean shouldRetryOn(Exception e) {
// Only retry on connection exceptions
return (e instanceof ClientHandlerException)
- && (e.getCause() instanceof ConnectException);
+ && (e.getCause() instanceof ConnectException ||
+ e.getCause() instanceof SocketTimeoutException);
}
};
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDFS-10869. Remove the unused
method InodeId#checkId(). Contributed by Jagadesh Kiran N
Posted by ji...@apache.org.
HDFS-10869. Remove the unused method InodeId#checkId(). Contributed by Jagadesh Kiran N
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3e37e243
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3e37e243
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3e37e243
Branch: refs/heads/MAPREDUCE-6608
Commit: 3e37e243ee041e843e060b17c622ab50c8f9ff11
Parents: 4d16d2d
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Sun Sep 25 11:02:46 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Sun Sep 25 11:02:46 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/INodeId.java | 16 ----------------
1 file changed, 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3e37e243/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
index 10139bf..16c28b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeId.java
@@ -17,10 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
-import java.io.FileNotFoundException;
-
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.util.SequentialNumber;
/**
@@ -39,19 +36,6 @@ public class INodeId extends SequentialNumber {
public static final long ROOT_INODE_ID = LAST_RESERVED_ID + 1;
public static final long INVALID_INODE_ID = -1;
- /**
- * To check if the request id is the same as saved id. Don't check fileId
- * with GRANDFATHER_INODE_ID for backward compatibility.
- */
- public static void checkId(long requestId, INode inode)
- throws FileNotFoundException {
- if (requestId != HdfsConstants.GRANDFATHER_INODE_ID && requestId != inode.getId()) {
- throw new FileNotFoundException(
- "ID mismatch. Request id and saved id: " + requestId + " , "
- + inode.getId() + " for file " + inode.getFullPathName());
- }
- }
-
INodeId() {
super(ROOT_INODE_ID);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-4973. YarnWebParams
next.fresh.interval should be next.refresh.interval (templedf via rkanter)
Posted by ji...@apache.org.
YARN-4973. YarnWebParams next.fresh.interval should be next.refresh.interval (templedf via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ffd4b7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ffd4b7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ffd4b7c
Branch: refs/heads/MAPREDUCE-6608
Commit: 5ffd4b7c1e1f3168483c708c7ed307a565389eb2
Parents: 4fc632a
Author: Robert Kanter <rk...@apache.org>
Authored: Thu Sep 22 16:45:34 2016 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Thu Sep 22 16:45:34 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ffd4b7c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 3792649..a34273c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -40,5 +40,5 @@ public interface YarnWebParams {
String NODE_STATE = "node.state";
String NODE_LABEL = "node.label";
String WEB_UI_TYPE = "web.ui.type";
- String NEXT_REFRESH_INTERVAL = "next.fresh.interval";
+ String NEXT_REFRESH_INTERVAL = "next.refresh.interval";
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: HADOOP-13621. s3:// should have
been fully cut off from trunk. Contributed by Mingliang Liu.
Posted by ji...@apache.org.
HADOOP-13621. s3:// should have been fully cut off from trunk. Contributed by Mingliang Liu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96142efa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96142efa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96142efa
Branch: refs/heads/MAPREDUCE-6608
Commit: 96142efa2dded3212d07e7e14e251066865fb7d2
Parents: f67237c
Author: Mingliang Liu <li...@apache.org>
Authored: Fri Sep 16 18:36:26 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Sat Sep 17 22:07:46 2016 -0700
----------------------------------------------------------------------
.../src/site/markdown/tools/hadoop-aws/index.md | 44 +++-----------------
1 file changed, 6 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96142efa/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 7fcadb9..160aa46 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -28,7 +28,7 @@ HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aws' in the list.
### Features
-**NOTE: `s3:` is being phased out. Use `s3n:` or `s3a:` instead.**
+**NOTE: `s3:` has been phased out. Use `s3n:` or `s3a:` instead.**
1. The second-generation, `s3n:` filesystem, making it easy to share
data between hadoop and other applications via the S3 object store.
@@ -86,38 +86,6 @@ these instructions \u2014and be aware that all issues related to S3 integration
in EMR can only be addressed by Amazon themselves: please raise your issues
with them.
-## S3
-
-The `s3://` filesystem is the original S3 store in the Hadoop codebase.
-It implements an inode-style filesystem atop S3, and was written to
-provide scaleability when S3 had significant limits on the size of blobs.
-It is incompatible with any other application's use of data in S3.
-
-It is now deprecated and will be removed in Hadoop 3. Please do not use,
-and migrate off data which is on it.
-
-### Dependencies
-
-* `jets3t` jar
-* `commons-codec` jar
-* `commons-logging` jar
-* `httpclient` jar
-* `httpcore` jar
-* `java-xmlbuilder` jar
-
-### Authentication properties
-
- <property>
- <name>fs.s3.awsAccessKeyId</name>
- <description>AWS access key ID</description>
- </property>
-
- <property>
- <name>fs.s3.awsSecretAccessKey</name>
- <description>AWS secret key</description>
- </property>
-
-
## S3N
S3N was the first S3 Filesystem client which used "native" S3 objects, hence
@@ -171,16 +139,16 @@ it should be used wherever possible.
### Other properties
<property>
- <name>fs.s3.buffer.dir</name>
+ <name>fs.s3n.buffer.dir</name>
<value>${hadoop.tmp.dir}/s3</value>
- <description>Determines where on the local filesystem the s3:/s3n: filesystem
+ <description>Determines where on the local filesystem the s3n: filesystem
should store files before sending them to S3
(or after retrieving them from S3).
</description>
</property>
<property>
- <name>fs.s3.maxRetries</name>
+ <name>fs.s3n.maxRetries</name>
<value>4</value>
<description>The maximum number of retries for reading or writing files to
S3, before we signal failure to the application.
@@ -188,7 +156,7 @@ it should be used wherever possible.
</property>
<property>
- <name>fs.s3.sleepTimeSeconds</name>
+ <name>fs.s3n.sleepTimeSeconds</name>
<value>10</value>
<description>The number of seconds to sleep between each S3 retry.
</description>
@@ -1011,7 +979,7 @@ includes `distcp`.
### `ClassNotFoundException: org.apache.hadoop.fs.s3a.S3AFileSystem`
-(or `org.apache.hadoop.fs.s3native.NativeS3FileSystem`, `org.apache.hadoop.fs.s3.S3FileSystem`).
+(or `org.apache.hadoop.fs.s3native.NativeS3FileSystem`).
These are the Hadoop classes, found in the `hadoop-aws` JAR. An exception
reporting one of these classes is missing means that this JAR is not on
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HADOOP-13602. Fix some warnings by
findbugs in hadoop-maven-plugin. (ozawa)
Posted by ji...@apache.org.
HADOOP-13602. Fix some warnings by findbugs in hadoop-maven-plugin. (ozawa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d619b48
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d619b48
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d619b48
Branch: refs/heads/MAPREDUCE-6608
Commit: 8d619b4896ac31f63fd0083594b6e7d207ef71a0
Parents: 537095d
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Fri Sep 23 01:37:06 2016 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Fri Sep 23 01:37:06 2016 +0900
----------------------------------------------------------------------
.../maven/plugin/cmakebuilder/CompileMojo.java | 4 +-
.../maven/plugin/cmakebuilder/TestMojo.java | 4 +-
.../hadoop/maven/plugin/protoc/ProtocMojo.java | 4 ++
.../apache/hadoop/maven/plugin/util/Exec.java | 6 ++-
.../plugin/versioninfo/VersionInfoMojo.java | 55 ++++++++++----------
5 files changed, 42 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d619b48/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java
index afb11cb..0196352 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/CompileMojo.java
@@ -14,6 +14,7 @@
package org.apache.hadoop.maven.plugin.cmakebuilder;
+import java.util.Locale;
import org.apache.hadoop.maven.plugin.util.Exec.OutputBufferThread;
import org.apache.hadoop.maven.plugin.util.Exec;
import org.apache.maven.plugin.AbstractMojo;
@@ -83,7 +84,8 @@ public class CompileMojo extends AbstractMojo {
// TODO: support Windows
private static void validatePlatform() throws MojoExecutionException {
- if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+ if (System.getProperty("os.name").toLowerCase(Locale.ENGLISH)
+ .startsWith("windows")) {
throw new MojoExecutionException("CMakeBuilder does not yet support " +
"the Windows platform.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d619b48/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/TestMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/TestMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/TestMojo.java
index e676efd..95b6264 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/TestMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/cmakebuilder/TestMojo.java
@@ -14,6 +14,7 @@
package org.apache.hadoop.maven.plugin.cmakebuilder;
+import java.util.Locale;
import org.apache.hadoop.maven.plugin.util.Exec;
import org.apache.maven.execution.MavenSession;
import org.apache.maven.plugin.AbstractMojo;
@@ -117,7 +118,8 @@ public class TestMojo extends AbstractMojo {
// TODO: support Windows
private static void validatePlatform() throws MojoExecutionException {
- if (System.getProperty("os.name").toLowerCase().startsWith("windows")) {
+ if (System.getProperty("os.name").toLowerCase(Locale.ENGLISH)
+ .startsWith("windows")) {
throw new MojoExecutionException("CMakeBuilder does not yet support " +
"the Windows platform.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d619b48/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
index 0dcac0e..df479fd 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/protoc/ProtocMojo.java
@@ -105,6 +105,10 @@ public class ProtocMojo extends AbstractMojo {
private boolean hasDirectoryChanged(File directory) throws IOException {
File[] listing = directory.listFiles();
boolean changed = false;
+ if (listing == null) {
+ // not changed.
+ return false;
+ }
// Do not exit early, since we need to compute and save checksums
// for each file within the directory.
for (File f : listing) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d619b48/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
index decfca4..5281631 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/util/Exec.java
@@ -84,8 +84,10 @@ public class Exec {
if (errors != null) {
errors.addAll(stdErr.getOutput());
}
- } catch (Exception ex) {
- mojo.getLog().warn(command + " failed: " + ex.toString());
+ } catch (IOException ioe) {
+ mojo.getLog().warn(command + " failed: " + ioe.toString());
+ } catch (InterruptedException ie) {
+ mojo.getLog().warn(command + " failed: " + ie.toString());
}
return retCode;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d619b48/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
index b6a45ec..cd2651b 100644
--- a/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
+++ b/hadoop-maven-plugins/src/main/java/org/apache/hadoop/maven/plugin/versioninfo/VersionInfoMojo.java
@@ -13,6 +13,7 @@
*/
package org.apache.hadoop.maven.plugin.versioninfo;
+import java.io.Serializable;
import java.util.Locale;
import org.apache.hadoop.maven.plugin.util.Exec;
import org.apache.hadoop.maven.plugin.util.FileSetUtils;
@@ -149,27 +150,23 @@ public class VersionInfoMojo extends AbstractMojo {
*/
private String[] getSvnUriInfo(String str) {
String[] res = new String[]{"Unknown", "Unknown"};
- try {
- String path = str;
- int index = path.indexOf("trunk");
+ String path = str;
+ int index = path.indexOf("trunk");
+ if (index > -1) {
+ res[0] = path.substring(0, index - 1);
+ res[1] = "trunk";
+ } else {
+ index = path.indexOf("branches");
if (index > -1) {
res[0] = path.substring(0, index - 1);
- res[1] = "trunk";
- } else {
- index = path.indexOf("branches");
+ int branchIndex = index + "branches".length() + 1;
+ index = path.indexOf("/", branchIndex);
if (index > -1) {
- res[0] = path.substring(0, index - 1);
- int branchIndex = index + "branches".length() + 1;
- index = path.indexOf("/", branchIndex);
- if (index > -1) {
- res[1] = path.substring(branchIndex, index);
- } else {
- res[1] = path.substring(branchIndex);
- }
+ res[1] = path.substring(branchIndex, index);
+ } else {
+ res[1] = path.substring(branchIndex);
}
}
- } catch (Exception ex) {
- getLog().warn("Could not determine URI & branch from SVN URI: " + str);
}
return res;
}
@@ -311,6 +308,20 @@ public class VersionInfoMojo extends AbstractMojo {
return sb.toString();
}
+ static class MD5Comparator implements Comparator<File>, Serializable {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public int compare(File lhs, File rhs) {
+ return normalizePath(lhs).compareTo(normalizePath(rhs));
+ }
+
+ private String normalizePath(File file) {
+ return file.getPath().toUpperCase(Locale.ENGLISH)
+ .replaceAll("\\\\", "/");
+ }
+ }
+
/**
* Computes and returns an MD5 checksum of the contents of all files in the
* input Maven FileSet.
@@ -323,17 +334,7 @@ public class VersionInfoMojo extends AbstractMojo {
// File order of MD5 calculation is significant. Sorting is done on
// unix-format names, case-folded, in order to get a platform-independent
// sort and calculate the same MD5 on all platforms.
- Collections.sort(files, new Comparator<File>() {
- @Override
- public int compare(File lhs, File rhs) {
- return normalizePath(lhs).compareTo(normalizePath(rhs));
- }
-
- private String normalizePath(File file) {
- return file.getPath().toUpperCase(Locale.ENGLISH)
- .replaceAll("\\\\", "/");
- }
- });
+ Collections.sort(files, new MD5Comparator());
byte[] md5 = computeMD5(files);
String md5str = byteArrayToString(md5);
getLog().info("Computed MD5: " + md5str);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-5637. Changes in NodeManager
to support Container rollback and commit. (asuresh)
Posted by ji...@apache.org.
YARN-5637. Changes in NodeManager to support Container rollback and commit. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3552c2b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3552c2b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3552c2b9
Branch: refs/heads/MAPREDUCE-6608
Commit: 3552c2b99dff4f21489ff284f9dcba40e897a1e5
Parents: ea839bd
Author: Arun Suresh <as...@apache.org>
Authored: Fri Sep 16 16:53:18 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Sun Sep 18 10:55:18 2016 -0700
----------------------------------------------------------------------
.../containermanager/ContainerManagerImpl.java | 68 ++++++-
.../containermanager/container/Container.java | 4 +
.../container/ContainerEventType.java | 1 +
.../container/ContainerImpl.java | 188 ++++++++++++++-----
.../container/ContainerReInitEvent.java | 20 +-
.../TestContainerManagerWithLCE.java | 42 ++++-
.../containermanager/TestContainerManager.java | 152 +++++++++++++--
.../nodemanager/webapp/MockContainer.java | 10 +
8 files changed, 401 insertions(+), 84 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index f909ca5..8a9ad99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -165,8 +165,8 @@ import static org.apache.hadoop.service.Service.STATE.STARTED;
public class ContainerManagerImpl extends CompositeService implements
ContainerManager {
- private enum ReinitOp {
- UPGRADE, COMMIT, ROLLBACK, LOCALIZE;
+ private enum ReInitOp {
+ RE_INIT, COMMIT, ROLLBACK, LOCALIZE;
}
/**
* Extra duration to wait for applications to be killed on shutdown.
@@ -1535,7 +1535,7 @@ public class ContainerManagerImpl extends CompositeService implements
ContainerId containerId = request.getContainerId();
Container container = preUpgradeOrLocalizeCheck(containerId,
- ReinitOp.LOCALIZE);
+ ReInitOp.LOCALIZE);
try {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
container.getResourceSet().addResources(request.getLocalResources());
@@ -1551,16 +1551,31 @@ public class ContainerManagerImpl extends CompositeService implements
return ResourceLocalizationResponse.newInstance();
}
- public void upgradeContainer(ContainerId containerId,
- ContainerLaunchContext upgradeLaunchContext) throws YarnException {
+ /**
+ * ReInitialize a container using a new Launch Context. If the
+ * retryFailureContext is not provided, The container is
+ * terminated on Failure.
+ *
+ * NOTE: Auto-Commit is true by default. This also means that the rollback
+ * context is purged as soon as the command to start the new process
+ * is sent. (The Container moves to RUNNING state)
+ *
+ * @param containerId Container Id.
+ * @param autoCommit Auto Commit flag.
+ * @param reInitLaunchContext Target Launch Context.
+ * @throws YarnException Yarn Exception.
+ */
+ public void reInitializeContainer(ContainerId containerId,
+ ContainerLaunchContext reInitLaunchContext, boolean autoCommit)
+ throws YarnException {
Container container = preUpgradeOrLocalizeCheck(containerId,
- ReinitOp.UPGRADE);
+ ReInitOp.RE_INIT);
ResourceSet resourceSet = new ResourceSet();
try {
- resourceSet.addResources(upgradeLaunchContext.getLocalResources());
+ resourceSet.addResources(reInitLaunchContext.getLocalResources());
dispatcher.getEventHandler().handle(
- new ContainerReInitEvent(containerId, upgradeLaunchContext,
- resourceSet));
+ new ContainerReInitEvent(containerId, reInitLaunchContext,
+ resourceSet, autoCommit));
container.setIsReInitializing(true);
} catch (URISyntaxException e) {
LOG.info("Error when parsing local resource URI for upgrade of" +
@@ -1569,8 +1584,41 @@ public class ContainerManagerImpl extends CompositeService implements
}
}
+ /**
+ * Rollback the last reInitialization, if possible.
+ * @param containerId Container ID.
+ * @throws YarnException Yarn Exception.
+ */
+ public void rollbackReInitialization(ContainerId containerId)
+ throws YarnException {
+ Container container = preUpgradeOrLocalizeCheck(containerId,
+ ReInitOp.ROLLBACK);
+ if (container.canRollback()) {
+ dispatcher.getEventHandler().handle(
+ new ContainerEvent(containerId, ContainerEventType.ROLLBACK_REINIT));
+ } else {
+ throw new YarnException("Nothing to rollback to !!");
+ }
+ }
+
+ /**
+ * Commit last reInitialization after which no rollback will be possible.
+ * @param containerId Container ID.
+ * @throws YarnException Yarn Exception.
+ */
+ public void commitReInitialization(ContainerId containerId)
+ throws YarnException {
+ Container container = preUpgradeOrLocalizeCheck(containerId,
+ ReInitOp.COMMIT);
+ if (container.canRollback()) {
+ container.commitUpgrade();
+ } else {
+ throw new YarnException("Nothing to Commit !!");
+ }
+ }
+
private Container preUpgradeOrLocalizeCheck(ContainerId containerId,
- ReinitOp op) throws YarnException {
+ ReInitOp op) throws YarnException {
Container container = context.getContainers().get(containerId);
if (container == null) {
throw new YarnException("Specified " + containerId + " does not exist!");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index f6c27ab..78c240a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -82,4 +82,8 @@ public interface Container extends EventHandler<ContainerEvent> {
void setIsReInitializing(boolean isReInitializing);
boolean isReInitializing();
+
+ boolean canRollback();
+
+ void commitUpgrade();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index 0b57505..afea0e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -26,6 +26,7 @@ public enum ContainerEventType {
UPDATE_DIAGNOSTICS_MSG,
CONTAINER_DONE,
REINITIALIZE_CONTAINER,
+ ROLLBACK_REINIT,
// DownloadManager
CONTAINER_INITED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 12bbea9..0707df0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -91,14 +91,42 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class ContainerImpl implements Container {
- private final static class ReInitializationContext {
- private final ResourceSet resourceSet;
+ private static final class ReInitializationContext {
private final ContainerLaunchContext newLaunchContext;
+ private final ResourceSet newResourceSet;
+
+ // Rollback state
+ private final ContainerLaunchContext oldLaunchContext;
+ private final ResourceSet oldResourceSet;
private ReInitializationContext(ContainerLaunchContext newLaunchContext,
- ResourceSet resourceSet) {
+ ResourceSet newResourceSet,
+ ContainerLaunchContext oldLaunchContext,
+ ResourceSet oldResourceSet) {
this.newLaunchContext = newLaunchContext;
- this.resourceSet = resourceSet;
+ this.newResourceSet = newResourceSet;
+ this.oldLaunchContext = oldLaunchContext;
+ this.oldResourceSet = oldResourceSet;
+ }
+
+ private boolean canRollback() {
+ return (oldLaunchContext != null);
+ }
+
+ private ResourceSet mergedResourceSet() {
+ if (oldLaunchContext == null) {
+ return newResourceSet;
+ }
+ return ResourceSet.merge(oldResourceSet, newResourceSet);
+ }
+
+ private ReInitializationContext createContextForRollback() {
+ if (oldLaunchContext == null) {
+ return null;
+ } else {
+ return new ReInitializationContext(
+ oldLaunchContext, oldResourceSet, null, null);
+ }
}
}
@@ -129,7 +157,7 @@ public class ContainerImpl implements Container {
private String logDir;
private String host;
private String ips;
- private ReInitializationContext reInitContext;
+ private volatile ReInitializationContext reInitContext;
private volatile boolean isReInitializing = false;
/** The NM-wide configuration - not specific to this container */
@@ -187,8 +215,8 @@ public class ContainerImpl implements Container {
}
// Configure the Retry Context
- this.containerRetryContext =
- configureRetryContext(conf, launchContext, this.containerId);
+ this.containerRetryContext = configureRetryContext(
+ conf, launchContext, this.containerId);
this.remainingRetryAttempts = this.containerRetryContext.getMaxRetries();
stateMachine = stateMachineFactory.make(this);
this.context = context;
@@ -320,12 +348,16 @@ public class ContainerImpl implements Container {
new ExitedWithSuccessTransition(true))
.addTransition(ContainerState.RUNNING,
EnumSet.of(ContainerState.RELAUNCHING,
+ ContainerState.LOCALIZED,
ContainerState.EXITED_WITH_FAILURE),
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new RetryFailureTransition())
.addTransition(ContainerState.RUNNING, ContainerState.REINITIALIZING,
ContainerEventType.REINITIALIZE_CONTAINER,
new ReInitializeContainerTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.REINITIALIZING,
+ ContainerEventType.ROLLBACK_REINIT,
+ new RollbackContainerTransition())
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
ContainerEventType.RESOURCE_LOCALIZED,
new ResourceLocalizedWhileRunningTransition())
@@ -884,15 +916,15 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked")
@Override
public void transition(ContainerImpl container, ContainerEvent event) {
- container.reInitContext = createReInitContext(event);
+ container.reInitContext = createReInitContext(container, event);
try {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>>
- pendingResources =
- container.reInitContext.resourceSet.getAllResourcesByVisibility();
- if (!pendingResources.isEmpty()) {
+ resByVisibility = container.reInitContext.newResourceSet
+ .getAllResourcesByVisibility();
+ if (!resByVisibility.isEmpty()) {
container.dispatcher.getEventHandler().handle(
new ContainerLocalizationRequestEvent(
- container, pendingResources));
+ container, resByVisibility));
} else {
// We are not waiting on any resources, so...
// Kill the current container.
@@ -909,10 +941,30 @@ public class ContainerImpl implements Container {
}
protected ReInitializationContext createReInitContext(
- ContainerEvent event) {
- ContainerReInitEvent rEvent = (ContainerReInitEvent)event;
- return new ReInitializationContext(rEvent.getReInitLaunchContext(),
- rEvent.getResourceSet());
+ ContainerImpl container, ContainerEvent event) {
+ ContainerReInitEvent reInitEvent = (ContainerReInitEvent)event;
+ return new ReInitializationContext(
+ reInitEvent.getReInitLaunchContext(),
+ reInitEvent.getResourceSet(),
+ // If AutoCommit is turned on, then no rollback can happen...
+ // So don't need to store the previous context.
+ (reInitEvent.isAutoCommit() ? null : container.launchContext),
+ (reInitEvent.isAutoCommit() ? null : container.resourceSet));
+ }
+ }
+
+ /**
+ * Transition to start the Rollback process.
+ */
+ static class RollbackContainerTransition extends
+ ReInitializeContainerTransition {
+
+ @Override
+ protected ReInitializationContext createReInitContext(ContainerImpl
+ container, ContainerEvent event) {
+ LOG.warn("Container [" + container.getContainerId() + "]" +
+ " about to be explicitly Rolledback !!");
+ return container.reInitContext.createContextForRollback();
}
}
@@ -928,10 +980,10 @@ public class ContainerImpl implements Container {
public void transition(ContainerImpl container, ContainerEvent event) {
ContainerResourceLocalizedEvent rsrcEvent =
(ContainerResourceLocalizedEvent) event;
- container.reInitContext.resourceSet.resourceLocalized(
+ container.reInitContext.newResourceSet.resourceLocalized(
rsrcEvent.getResource(), rsrcEvent.getLocation());
// Check if all ResourceLocalization has completed
- if (container.reInitContext.resourceSet.getPendingResources()
+ if (container.reInitContext.newResourceSet.getPendingResources()
.isEmpty()) {
// Kill the current container.
container.dispatcher.getEventHandler().handle(
@@ -1028,10 +1080,13 @@ public class ContainerImpl implements Container {
container.metrics.runningContainer();
container.wasLaunched = true;
- if (container.reInitContext != null) {
+ container.setIsReInitializing(false);
+ // Check if this launch was due to a re-initialization.
+ // If autocommit == true, then wipe the re-init context. This ensures
+ // that any subsequent failures do not trigger a rollback.
+ if (container.reInitContext != null
+ && !container.reInitContext.canRollback()) {
container.reInitContext = null;
- // Set rollback context here..
- container.setIsReInitializing(false);
}
if (container.recoveredAsKilled) {
@@ -1148,36 +1203,50 @@ public class ContainerImpl implements Container {
+ container.getContainerId(), e);
}
}
- LOG.info("Relaunching Container " + container.getContainerId()
- + ". Remaining retry attempts(after relaunch) : "
- + container.remainingRetryAttempts
- + ". Interval between retries is "
- + container.containerRetryContext.getRetryInterval() + "ms");
- container.wasLaunched = false;
- container.metrics.endRunningContainer();
- if (container.containerRetryContext.getRetryInterval() == 0) {
- container.sendRelaunchEvent();
- } else {
- // wait for some time, then send launch event
- new Thread() {
- @Override
- public void run() {
- try {
- Thread.sleep(
- container.containerRetryContext.getRetryInterval());
- container.sendRelaunchEvent();
- } catch (InterruptedException e) {
- return;
- }
- }
- }.start();
- }
+ doRelaunch(container, container.remainingRetryAttempts,
+ container.containerRetryContext.getRetryInterval());
return ContainerState.RELAUNCHING;
+ } else if (container.canRollback()) {
+ // Rollback is possible only if the previous launch context is
+ // available.
+ container.addDiagnostics("Container Re-init Auto Rolled-Back.");
+ LOG.info("Rolling back Container reInitialization for [" +
+ container.getContainerId() + "] !!");
+ container.reInitContext =
+ container.reInitContext.createContextForRollback();
+ new KilledForReInitializationTransition().transition(container, event);
+ return ContainerState.LOCALIZED;
} else {
new ExitedWithFailureTransition(true).transition(container, event);
return ContainerState.EXITED_WITH_FAILURE;
}
}
+
+ private void doRelaunch(final ContainerImpl container,
+ int remainingRetryAttempts, final int retryInterval) {
+ LOG.info("Relaunching Container " + container.getContainerId()
+ + ". Remaining retry attempts(after relaunch) : "
+ + remainingRetryAttempts + ". Interval between retries is "
+ + retryInterval + "ms");
+ container.wasLaunched = false;
+ container.metrics.endRunningContainer();
+ if (retryInterval == 0) {
+ container.sendRelaunchEvent();
+ } else {
+ // wait for some time, then send launch event
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ Thread.sleep(retryInterval);
+ container.sendRelaunchEvent();
+ } catch (InterruptedException e) {
+ return;
+ }
+ }
+ }.start();
+ }
+ }
}
@Override
@@ -1188,24 +1257,29 @@ public class ContainerImpl implements Container {
@Override
public boolean shouldRetry(int errorCode) {
+ return shouldRetry(errorCode, containerRetryContext,
+ remainingRetryAttempts);
+ }
+
+ public static boolean shouldRetry(int errorCode,
+ ContainerRetryContext retryContext, int remainingRetryAttempts) {
if (errorCode == ExitCode.SUCCESS.getExitCode()
|| errorCode == ExitCode.FORCE_KILLED.getExitCode()
|| errorCode == ExitCode.TERMINATED.getExitCode()) {
return false;
}
- ContainerRetryPolicy retryPolicy = containerRetryContext.getRetryPolicy();
+ ContainerRetryPolicy retryPolicy = retryContext.getRetryPolicy();
if (retryPolicy == ContainerRetryPolicy.RETRY_ON_ALL_ERRORS
|| (retryPolicy == ContainerRetryPolicy.RETRY_ON_SPECIFIC_ERROR_CODES
- && containerRetryContext.getErrorCodes() != null
- && containerRetryContext.getErrorCodes().contains(errorCode))) {
+ && retryContext.getErrorCodes() != null
+ && retryContext.getErrorCodes().contains(errorCode))) {
return remainingRetryAttempts > 0
|| remainingRetryAttempts == ContainerRetryContext.RETRY_FOREVER;
}
return false;
}
-
/**
* Transition to EXITED_WITH_FAILURE
*/
@@ -1240,13 +1314,12 @@ public class ContainerImpl implements Container {
// Re configure the Retry Context
container.containerRetryContext =
configureRetryContext(container.context.getConf(),
- container.launchContext, container.containerId);
+ container.launchContext, container.containerId);
// Reset the retry attempts since its a fresh start
container.remainingRetryAttempts =
container.containerRetryContext.getMaxRetries();
- container.resourceSet = ResourceSet.merge(
- container.resourceSet, container.reInitContext.resourceSet);
+ container.resourceSet = container.reInitContext.mergedResourceSet();
container.sendLaunchEvent();
}
@@ -1589,4 +1662,15 @@ public class ContainerImpl implements Container {
public boolean isReInitializing() {
return this.isReInitializing;
}
+
+ @Override
+ public boolean canRollback() {
+ return (this.reInitContext != null)
+ && (this.reInitContext.canRollback());
+ }
+
+ @Override
+ public void commitUpgrade() {
+ this.reInitContext = null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerReInitEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerReInitEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerReInitEvent.java
index 2ccdbd7..46eba03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerReInitEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerReInitEvent.java
@@ -30,18 +30,22 @@ public class ContainerReInitEvent extends ContainerEvent {
private final ContainerLaunchContext reInitLaunchContext;
private final ResourceSet resourceSet;
+ private final boolean autoCommit;
/**
* Container Re-Init Event.
- * @param cID Container Id
- * @param upgradeContext Upgrade context
- * @param resourceSet Resource Set
+ * @param cID Container Id.
+ * @param upgradeContext Upgrade Context.
+ * @param resourceSet Resource Set.
+ * @param autoCommit Auto Commit.
*/
public ContainerReInitEvent(ContainerId cID,
- ContainerLaunchContext upgradeContext, ResourceSet resourceSet){
+ ContainerLaunchContext upgradeContext,
+ ResourceSet resourceSet, boolean autoCommit){
super(cID, ContainerEventType.REINITIALIZE_CONTAINER);
this.reInitLaunchContext = upgradeContext;
this.resourceSet = resourceSet;
+ this.autoCommit = autoCommit;
}
/**
@@ -59,4 +63,12 @@ public class ContainerReInitEvent extends ContainerEvent {
public ResourceSet getResourceSet() {
return resourceSet;
}
+
+ /**
+ * Should this re-Initialization be auto-committed.
+ * @return AutoCommit.
+ */
+ public boolean isAutoCommit() {
+ return autoCommit;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 8a27849..79182ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -270,15 +270,15 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
}
@Override
- public void testContainerUpgradeSuccess() throws IOException,
+ public void testContainerUpgradeSuccessAutoCommit() throws IOException,
InterruptedException, YarnException {
// Don't run the test if the binary is not available.
if (!shouldRunTest()) {
LOG.info("LCE binary path is not passed. Not running the test");
return;
}
- LOG.info("Running testContainerUpgradeSuccess");
- super.testContainerUpgradeSuccess();
+ LOG.info("Running testContainerUpgradeSuccessAutoCommit");
+ super.testContainerUpgradeSuccessAutoCommit();
}
@Override
@@ -294,6 +294,42 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
}
@Override
+ public void testContainerUpgradeSuccessExplicitCommit() throws IOException,
+ InterruptedException, YarnException {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testContainerUpgradeSuccessExplicitCommit");
+ super.testContainerUpgradeSuccessExplicitCommit();
+ }
+
+ @Override
+ public void testContainerUpgradeSuccessExplicitRollback() throws IOException,
+ InterruptedException, YarnException {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testContainerUpgradeSuccessExplicitRollback");
+ super.testContainerUpgradeSuccessExplicitRollback();
+ }
+
+ @Override
+ public void testContainerUpgradeRollbackDueToFailure() throws IOException,
+ InterruptedException, YarnException {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ LOG.info("Running testContainerUpgradeRollbackDueToFailure");
+ super.testContainerUpgradeRollbackDueToFailure();
+ }
+
+ @Override
public void testContainerUpgradeProcessFailure() throws IOException,
InterruptedException, YarnException {
// Don't run the test if the binary is not available.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 843dc2a..72049e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -369,9 +369,8 @@ public class TestContainerManager extends BaseContainerManagerTest {
DefaultContainerExecutor.containerIsAlive(pid));
}
- @Test
- public void testContainerUpgradeSuccess() throws IOException,
- InterruptedException, YarnException {
+ private String[] testContainerUpgradeSuccess(boolean autoCommit)
+ throws IOException, InterruptedException, YarnException {
containerManager.start();
// ////// Construct the Container-id
ContainerId cId = createContainerId(0);
@@ -381,7 +380,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
File newStartFile = new File(tmpDir, "start_file_n.txt").getAbsoluteFile();
- prepareContainerUpgrade(false, false, cId, newStartFile);
+ prepareContainerUpgrade(autoCommit, false, false, cId, newStartFile);
// Assert that the First process is not alive anymore
Assert.assertFalse("Process is still alive!",
@@ -407,6 +406,80 @@ public class TestContainerManager extends BaseContainerManagerTest {
// Assert that the New process is alive
Assert.assertTrue("New Process is not alive!",
DefaultContainerExecutor.containerIsAlive(newPid));
+ return new String[]{pid, newPid};
+ }
+
+ @Test
+ public void testContainerUpgradeSuccessAutoCommit() throws IOException,
+ InterruptedException, YarnException {
+ testContainerUpgradeSuccess(true);
+ // Should not be able to Commit (since already auto committed)
+ try {
+ containerManager.commitReInitialization(createContainerId(0));
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("Nothing to Commit"));
+ }
+ }
+
+ @Test
+ public void testContainerUpgradeSuccessExplicitCommit() throws IOException,
+ InterruptedException, YarnException {
+ testContainerUpgradeSuccess(false);
+ ContainerId cId = createContainerId(0);
+ containerManager.commitReInitialization(cId);
+ // Should not be able to Rollback once committed
+ try {
+ containerManager.rollbackReInitialization(cId);
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e.getMessage().contains("Nothing to rollback to"));
+ }
+ }
+
+ @Test
+ public void testContainerUpgradeSuccessExplicitRollback() throws IOException,
+ InterruptedException, YarnException {
+ String[] pids = testContainerUpgradeSuccess(false);
+
+ // Delete the old start File..
+ File oldStartFile = new File(tmpDir, "start_file_o.txt").getAbsoluteFile();
+ oldStartFile.delete();
+
+ ContainerId cId = createContainerId(0);
+ // Explicit Rollback
+ containerManager.rollbackReInitialization(cId);
+
+ // Original should be dead anyway
+ Assert.assertFalse("Original Process is still alive!",
+ DefaultContainerExecutor.containerIsAlive(pids[0]));
+
+ // Wait for upgraded process to die
+ int timeoutSecs = 0;
+ while (!DefaultContainerExecutor.containerIsAlive(pids[1])
+ && timeoutSecs++ < 20) {
+ Thread.sleep(1000);
+ LOG.info("Waiting for Upgraded process to die..");
+ }
+
+ timeoutSecs = 0;
+ // Wait for new processStartfile to be created
+ while (!oldStartFile.exists() && timeoutSecs++ < 20) {
+ Thread.sleep(1000);
+ LOG.info("Waiting for New process start-file to be created");
+ }
+
+ // Now verify the contents of the file
+ BufferedReader reader =
+ new BufferedReader(new FileReader(oldStartFile));
+ Assert.assertEquals("Hello World!", reader.readLine());
+ // Get the pid of the process
+ String rolledBackPid = reader.readLine().trim();
+ // No more lines
+ Assert.assertEquals(null, reader.readLine());
+
+ Assert.assertNotEquals("The Rolled-back process should be a different pid",
+ pids[0], rolledBackPid);
}
@Test
@@ -424,7 +497,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
File newStartFile = new File(tmpDir, "start_file_n.txt").getAbsoluteFile();
- prepareContainerUpgrade(true, true, cId, newStartFile);
+ prepareContainerUpgrade(false, true, true, cId, newStartFile);
// Assert that the First process is STILL alive
// since upgrade was terminated..
@@ -447,22 +520,69 @@ public class TestContainerManager extends BaseContainerManagerTest {
File newStartFile = new File(tmpDir, "start_file_n.txt").getAbsoluteFile();
- prepareContainerUpgrade(true, false, cId, newStartFile);
+ // Since Autocommit is true, there is also no rollback context...
+ // which implies that if the new process fails, since there is no
+ // rollback, it is terminated.
+ prepareContainerUpgrade(true, true, false, cId, newStartFile);
// Assert that the First process is not alive anymore
Assert.assertFalse("Process is still alive!",
DefaultContainerExecutor.containerIsAlive(pid));
}
+ @Test
+ public void testContainerUpgradeRollbackDueToFailure() throws IOException,
+ InterruptedException, YarnException {
+ if (Shell.WINDOWS) {
+ return;
+ }
+ containerManager.start();
+ // ////// Construct the Container-id
+ ContainerId cId = createContainerId(0);
+ File oldStartFile = new File(tmpDir, "start_file_o.txt").getAbsoluteFile();
+
+ String pid = prepareInitialContainer(cId, oldStartFile);
+
+ File newStartFile = new File(tmpDir, "start_file_n.txt").getAbsoluteFile();
+
+ prepareContainerUpgrade(false, true, false, cId, newStartFile);
+
+ // Assert that the First process is not alive anymore
+ Assert.assertFalse("Original Process is still alive!",
+ DefaultContainerExecutor.containerIsAlive(pid));
+
+ int timeoutSecs = 0;
+ // Wait for oldStartFile to be created
+ while (!oldStartFile.exists() && timeoutSecs++ < 20) {
+ System.out.println("\nFiles: " +
+ Arrays.toString(oldStartFile.getParentFile().list()));
+ Thread.sleep(1000);
+ LOG.info("Waiting for New process start-file to be created");
+ }
+
+ // Now verify the contents of the file
+ BufferedReader reader =
+ new BufferedReader(new FileReader(oldStartFile));
+ Assert.assertEquals("Hello World!", reader.readLine());
+ // Get the pid of the process
+ String rolledBackPid = reader.readLine().trim();
+ // No more lines
+ Assert.assertEquals(null, reader.readLine());
+
+ Assert.assertNotEquals("The Rolled-back process should be a different pid",
+ pid, rolledBackPid);
+ }
+
/**
* Prepare a launch Context for container upgrade and request the
* Container Manager to re-initialize a running container using the
* new launch context.
+ * @param autoCommit Enable autoCommit.
* @param failCmd injects a start script that intentionally fails.
* @param failLoc injects a bad file Location that will fail localization.
*/
- private void prepareContainerUpgrade(boolean failCmd, boolean failLoc,
- ContainerId cId, File startFile)
+ private void prepareContainerUpgrade(boolean autoCommit, boolean failCmd,
+ boolean failLoc, ContainerId cId, File startFile)
throws FileNotFoundException, YarnException, InterruptedException {
// Re-write scriptfile and processStartFile
File scriptFile = Shell.appendScriptExtension(tmpDir, "scriptFile_new");
@@ -471,13 +591,15 @@ public class TestContainerManager extends BaseContainerManagerTest {
writeScriptFile(fileWriter, "Upgrade World!", startFile, cId, failCmd);
ContainerLaunchContext containerLaunchContext =
- prepareContainerLaunchContext(scriptFile, "dest_file_new", failLoc);
+ prepareContainerLaunchContext(scriptFile, "dest_file_new", failLoc, 0);
- containerManager.upgradeContainer(cId, containerLaunchContext);
+ containerManager.reInitializeContainer(cId, containerLaunchContext,
+ autoCommit);
try {
- containerManager.upgradeContainer(cId, containerLaunchContext);
+ containerManager.reInitializeContainer(cId, containerLaunchContext,
+ autoCommit);
} catch (Exception e) {
- Assert.assertTrue(e.getMessage().contains("Cannot perform UPGRADE"));
+ Assert.assertTrue(e.getMessage().contains("Cannot perform RE_INIT"));
}
int timeoutSecs = 0;
int maxTimeToWait = failLoc ? 10 : 20;
@@ -501,7 +623,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
writeScriptFile(fileWriterOld, "Hello World!", startFile, cId, false);
ContainerLaunchContext containerLaunchContext =
- prepareContainerLaunchContext(scriptFileOld, "dest_file", false);
+ prepareContainerLaunchContext(scriptFileOld, "dest_file", false, 4);
StartContainerRequest scRequest =
StartContainerRequest.newInstance(containerLaunchContext,
@@ -562,7 +684,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
}
private ContainerLaunchContext prepareContainerLaunchContext(File scriptFile,
- String destFName, boolean putBadFile) {
+ String destFName, boolean putBadFile, int numRetries) {
ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
URL resourceAlpha = null;
@@ -592,7 +714,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
ContainerRetryContext containerRetryContext = ContainerRetryContext
.newInstance(
ContainerRetryPolicy.RETRY_ON_SPECIFIC_ERROR_CODES,
- new HashSet<>(Arrays.asList(Integer.valueOf(111))), 4, 0);
+ new HashSet<>(Arrays.asList(Integer.valueOf(111))), numRetries, 0);
containerLaunchContext.setContainerRetryContext(containerRetryContext);
List<String> commands = Arrays.asList(
Shell.getRunScriptCommand(scriptFile));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3552c2b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 8c8bec7..164488d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -205,4 +205,14 @@ public class MockContainer implements Container {
public boolean isReInitializing() {
return false;
}
+
+ @Override
+ public boolean canRollback() {
+ return false;
+ }
+
+ @Override
+ public void commitUpgrade() {
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: HDFS-10886. Replace
fs.default.name with fs.defaultFS in viewfs document.
Posted by ji...@apache.org.
HDFS-10886. Replace fs.default.name with fs.defaultFS in viewfs document.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bbdf350f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bbdf350f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bbdf350f
Branch: refs/heads/MAPREDUCE-6608
Commit: bbdf350ff9fb624fe736c1eb9271c5dcb4e14b06
Parents: c4480f7
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Sep 23 19:21:34 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Sep 23 19:21:34 2016 +0530
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbdf350f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
index 23465a6..3beda53 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/viewfs/ViewFs.java
@@ -83,7 +83,7 @@ import org.apache.hadoop.util.Time;
* ViewFs is specified with the following URI: <b>viewfs:///</b>
* <p>
* To use viewfs one would typically set the default file system in the
- * config (i.e. fs.default.name< = viewfs:///) along with the
+ * config (i.e. fs.defaultFS < = viewfs:///) along with the
* mount table config variables as described below.
*
* <p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bbdf350f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
index 5f88def..ddec9b0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/ViewFs.md
@@ -108,7 +108,7 @@ The mount points of a mount table are specified in the standard Hadoop configura
```xml
<property>
- <name>fs.default.name</name>
+ <name>fs.defaultFS</name>
<value>viewfs://clusterX</value>
</property>
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-10861. Refactor StripeReaders
and use ECChunk version decode API. Contributed by Sammi Chen
Posted by ji...@apache.org.
HDFS-10861. Refactor StripeReaders and use ECChunk version decode API. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/734d54c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/734d54c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/734d54c1
Branch: refs/heads/MAPREDUCE-6608
Commit: 734d54c1a8950446e68098f62d8964e02ecc2890
Parents: 2b66d9e
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Sep 21 21:34:48 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Sep 21 21:34:48 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/io/ElasticByteBufferPool.java | 2 +-
.../apache/hadoop/io/erasurecode/ECChunk.java | 22 +
.../io/erasurecode/rawcoder/CoderUtil.java | 3 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 20 +-
.../hadoop/hdfs/DFSStripedInputStream.java | 654 +++----------------
.../hadoop/hdfs/PositionStripeReader.java | 104 +++
.../hadoop/hdfs/StatefulStripeReader.java | 95 +++
.../org/apache/hadoop/hdfs/StripeReader.java | 463 +++++++++++++
.../hadoop/hdfs/util/StripedBlockUtil.java | 158 ++---
.../hadoop/hdfs/util/TestStripedBlockUtil.java | 1 -
10 files changed, 844 insertions(+), 678 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
index c35d608..023f37f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/ElasticByteBufferPool.java
@@ -85,7 +85,7 @@ public final class ElasticByteBufferPool implements ByteBufferPool {
private final TreeMap<Key, ByteBuffer> getBufferTree(boolean direct) {
return direct ? directBuffers : buffers;
}
-
+
@Override
public synchronized ByteBuffer getBuffer(boolean direct, int length) {
TreeMap<Key, ByteBuffer> tree = getBufferTree(direct);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
index cd7c6be..536715b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ECChunk.java
@@ -29,6 +29,9 @@ public class ECChunk {
private ByteBuffer chunkBuffer;
+ // TODO: should be in a more general flags
+ private boolean allZero = false;
+
/**
* Wrapping a ByteBuffer
* @param buffer buffer to be wrapped by the chunk
@@ -37,6 +40,13 @@ public class ECChunk {
this.chunkBuffer = buffer;
}
+ public ECChunk(ByteBuffer buffer, int offset, int len) {
+ ByteBuffer tmp = buffer.duplicate();
+ tmp.position(offset);
+ tmp.limit(offset + len);
+ this.chunkBuffer = tmp.slice();
+ }
+
/**
* Wrapping a bytes array
* @param buffer buffer to be wrapped by the chunk
@@ -45,6 +55,18 @@ public class ECChunk {
this.chunkBuffer = ByteBuffer.wrap(buffer);
}
+ public ECChunk(byte[] buffer, int offset, int len) {
+ this.chunkBuffer = ByteBuffer.wrap(buffer, offset, len);
+ }
+
+ public boolean isAllZero() {
+ return allZero;
+ }
+
+ public void setAllZero(boolean allZero) {
+ this.allZero = allZero;
+ }
+
/**
* Convert to ByteBuffer
* @return ByteBuffer
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java
index b22d44f..ef34639 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/rawcoder/CoderUtil.java
@@ -115,6 +115,9 @@ final class CoderUtil {
buffers[i] = null;
} else {
buffers[i] = chunk.getBuffer();
+ if (chunk.isAllZero()) {
+ CoderUtil.resetBuffer(buffers[i], buffers[i].remaining());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 31fa897..dbffc64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -240,7 +240,7 @@ public class DFSInputStream extends FSInputStream
Iterator<LocatedBlock> oldIter = locatedBlocks.getLocatedBlocks().iterator();
Iterator<LocatedBlock> newIter = newInfo.getLocatedBlocks().iterator();
while (oldIter.hasNext() && newIter.hasNext()) {
- if (! oldIter.next().getBlock().equals(newIter.next().getBlock())) {
+ if (!oldIter.next().getBlock().equals(newIter.next().getBlock())) {
throw new IOException("Blocklist for " + src + " has changed!");
}
}
@@ -677,8 +677,8 @@ public class DFSInputStream extends FSInputStream
if (oneByteBuf == null) {
oneByteBuf = new byte[1];
}
- int ret = read( oneByteBuf, 0, 1 );
- return ( ret <= 0 ) ? -1 : (oneByteBuf[0] & 0xff);
+ int ret = read(oneByteBuf, 0, 1);
+ return (ret <= 0) ? -1 : (oneByteBuf[0] & 0xff);
}
/* This is a used by regular read() and handles ChecksumExceptions.
@@ -702,7 +702,7 @@ public class DFSInputStream extends FSInputStream
// retry as many times as seekToNewSource allows.
try {
return reader.readFromBlock(blockReader, len);
- } catch ( ChecksumException ce ) {
+ } catch (ChecksumException ce) {
DFSClient.LOG.warn("Found Checksum error for "
+ getCurrentBlock() + " from " + currentNode
+ " at " + ce.getPos());
@@ -710,7 +710,7 @@ public class DFSInputStream extends FSInputStream
retryCurrentNode = false;
// we want to remember which block replicas we have tried
corruptedBlocks.addCorruptedBlock(getCurrentBlock(), currentNode);
- } catch ( IOException e ) {
+ } catch (IOException e) {
if (!retryCurrentNode) {
DFSClient.LOG.warn("Exception while reading from "
+ getCurrentBlock() + " of " + src + " from "
@@ -779,7 +779,9 @@ public class DFSInputStream extends FSInputStream
DFSClient.LOG.warn("DFS Read", e);
}
blockEnd = -1;
- if (currentNode != null) { addToDeadNodes(currentNode); }
+ if (currentNode != null) {
+ addToDeadNodes(currentNode);
+ }
if (--retries == 0) {
throw e;
}
@@ -1397,10 +1399,10 @@ public class DFSInputStream extends FSInputStream
@Override
public long skip(long n) throws IOException {
- if ( n > 0 ) {
+ if (n > 0) {
long curPos = getPos();
long fileLen = getFileLength();
- if( n+curPos > fileLen ) {
+ if (n+curPos > fileLen) {
n = fileLen - curPos;
}
seek(curPos+n);
@@ -1550,7 +1552,7 @@ public class DFSInputStream extends FSInputStream
* Get statistics about the reads which this DFSInputStream has done.
*/
public ReadStatistics getReadStatistics() {
- return new ReadStatistics(readStatistics);
+ return readStatistics;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
index ccaf6a7..922f74e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSStripedInputStream.java
@@ -17,24 +17,21 @@
*/
package org.apache.hadoop.hdfs;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.ReadOption;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.InvalidEncryptionKeyException;
import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
+import org.apache.hadoop.hdfs.StripeReader.BlockReaderInfo;
+import org.apache.hadoop.hdfs.StripeReader.ReaderRetryPolicy;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripeRange;
import org.apache.hadoop.io.ByteBufferPool;
-import static org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
-import static org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunk;
-import static org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult;
-
import org.apache.hadoop.io.ElasticByteBufferPool;
import org.apache.hadoop.io.erasurecode.CodecUtil;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
@@ -44,7 +41,6 @@ import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
import java.io.EOFException;
import java.io.IOException;
-import java.io.InterruptedIOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
@@ -53,111 +49,32 @@ import java.util.EnumSet;
import java.util.List;
import java.util.Set;
import java.util.Collection;
-import java.util.Map;
-import java.util.HashMap;
-import java.util.concurrent.CompletionService;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorCompletionService;
-import java.util.concurrent.Callable;
-import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
/**
- * DFSStripedInputStream reads from striped block groups
+ * DFSStripedInputStream reads from striped block groups.
*/
@InterfaceAudience.Private
public class DFSStripedInputStream extends DFSInputStream {
- private static class ReaderRetryPolicy {
- private int fetchEncryptionKeyTimes = 1;
- private int fetchTokenTimes = 1;
-
- void refetchEncryptionKey() {
- fetchEncryptionKeyTimes--;
- }
-
- void refetchToken() {
- fetchTokenTimes--;
- }
-
- boolean shouldRefetchEncryptionKey() {
- return fetchEncryptionKeyTimes > 0;
- }
-
- boolean shouldRefetchToken() {
- return fetchTokenTimes > 0;
- }
- }
-
- /** Used to indicate the buffered data's range in the block group */
- private static class StripeRange {
- /** start offset in the block group (inclusive) */
- final long offsetInBlock;
- /** length of the stripe range */
- final long length;
-
- StripeRange(long offsetInBlock, long length) {
- Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
- this.offsetInBlock = offsetInBlock;
- this.length = length;
- }
-
- boolean include(long pos) {
- return pos >= offsetInBlock && pos < offsetInBlock + length;
- }
- }
-
- private static class BlockReaderInfo {
- final BlockReader reader;
- final DatanodeInfo datanode;
- /**
- * when initializing block readers, their starting offsets are set to the same
- * number: the smallest internal block offsets among all the readers. This is
- * because it is possible that for some internal blocks we have to read
- * "backwards" for decoding purpose. We thus use this offset array to track
- * offsets for all the block readers so that we can skip data if necessary.
- */
- long blockReaderOffset;
- /**
- * We use this field to indicate whether we should use this reader. In case
- * we hit any issue with this reader, we set this field to true and avoid
- * using it for the next stripe.
- */
- boolean shouldSkip = false;
-
- BlockReaderInfo(BlockReader reader, DatanodeInfo dn, long offset) {
- this.reader = reader;
- this.datanode = dn;
- this.blockReaderOffset = offset;
- }
-
- void setOffset(long offset) {
- this.blockReaderOffset = offset;
- }
-
- void skip() {
- this.shouldSkip = true;
- }
- }
-
private static final ByteBufferPool BUFFER_POOL = new ElasticByteBufferPool();
-
private final BlockReaderInfo[] blockReaders;
private final int cellSize;
private final short dataBlkNum;
private final short parityBlkNum;
private final int groupSize;
- /** the buffer for a complete stripe */
+ /** the buffer for a complete stripe. */
private ByteBuffer curStripeBuf;
private ByteBuffer parityBuf;
private final ErasureCodingPolicy ecPolicy;
private final RawErasureDecoder decoder;
/**
- * indicate the start/end offset of the current buffered stripe in the
- * block group
+ * Indicate the start/end offset of the current buffered stripe in the
+ * block group.
*/
private StripeRange curStripeRange;
- private final CompletionService<Void> readingService;
/**
* When warning the user of a lost block in striping mode, we remember the
@@ -167,8 +84,8 @@ public class DFSStripedInputStream extends DFSInputStream {
*
* To minimize the overhead, we only store the datanodeUuid in this set
*/
- private final Set<String> warnedNodes = Collections.newSetFromMap(
- new ConcurrentHashMap<String, Boolean>());
+ private final Set<String> warnedNodes =
+ Collections.newSetFromMap(new ConcurrentHashMap<>());
DFSStripedInputStream(DFSClient dfsClient, String src,
boolean verifyChecksum, ErasureCodingPolicy ecPolicy,
@@ -183,8 +100,6 @@ public class DFSStripedInputStream extends DFSInputStream {
groupSize = dataBlkNum + parityBlkNum;
blockReaders = new BlockReaderInfo[groupSize];
curStripeRange = new StripeRange(0, 0);
- readingService =
- new ExecutorCompletionService<>(dfsClient.getStripedReadsThreadPool());
ErasureCoderOptions coderOptions = new ErasureCoderOptions(
dataBlkNum, parityBlkNum);
decoder = CodecUtil.createRawDecoder(dfsClient.getConfiguration(),
@@ -198,7 +113,7 @@ public class DFSStripedInputStream extends DFSInputStream {
return decoder.preferDirectBuffer();
}
- private void resetCurStripeBuffer() {
+ void resetCurStripeBuffer() {
if (curStripeBuf == null) {
curStripeBuf = BUFFER_POOL.getBuffer(useDirectBuffer(),
cellSize * dataBlkNum);
@@ -207,7 +122,7 @@ public class DFSStripedInputStream extends DFSInputStream {
curStripeRange = new StripeRange(0, 0);
}
- private ByteBuffer getParityBuffer() {
+ protected ByteBuffer getParityBuffer() {
if (parityBuf == null) {
parityBuf = BUFFER_POOL.getBuffer(useDirectBuffer(),
cellSize * parityBlkNum);
@@ -216,6 +131,29 @@ public class DFSStripedInputStream extends DFSInputStream {
return parityBuf;
}
+ protected ByteBuffer getCurStripeBuf() {
+ return curStripeBuf;
+ }
+
+ protected String getSrc() {
+ return src;
+ }
+
+ protected DFSClient getDFSClient() {
+ return dfsClient;
+ }
+
+ protected LocatedBlocks getLocatedBlocks() {
+ return locatedBlocks;
+ }
+
+ protected ByteBufferPool getBufferPool() {
+ return BUFFER_POOL;
+ }
+
+ protected ThreadPoolExecutor getStripedReadsThreadPool(){
+ return dfsClient.getStripedReadsThreadPool();
+ }
/**
* When seeking into a new block group, create blockReader for each internal
* block in the group.
@@ -268,7 +206,7 @@ public class DFSStripedInputStream extends DFSInputStream {
blockEnd = -1;
}
- private void closeReader(BlockReaderInfo readerInfo) {
+ protected void closeReader(BlockReaderInfo readerInfo) {
if (readerInfo != null) {
if (readerInfo.reader != null) {
try {
@@ -288,6 +226,59 @@ public class DFSStripedInputStream extends DFSInputStream {
return pos - currentLocatedBlock.getStartOffset();
}
+ boolean createBlockReader(LocatedBlock block, long offsetInBlock,
+ LocatedBlock[] targetBlocks, BlockReaderInfo[] readerInfos,
+ int chunkIndex) throws IOException {
+ BlockReader reader = null;
+ final ReaderRetryPolicy retry = new ReaderRetryPolicy();
+ DFSInputStream.DNAddrPair dnInfo =
+ new DFSInputStream.DNAddrPair(null, null, null);
+
+ while (true) {
+ try {
+ // the cached block location might have been re-fetched, so always
+ // get it from cache.
+ block = refreshLocatedBlock(block);
+ targetBlocks[chunkIndex] = block;
+
+ // internal block has one location, just rule out the deadNodes
+ dnInfo = getBestNodeDNAddrPair(block, null);
+ if (dnInfo == null) {
+ break;
+ }
+ reader = getBlockReader(block, offsetInBlock,
+ block.getBlockSize() - offsetInBlock,
+ dnInfo.addr, dnInfo.storageType, dnInfo.info);
+ } catch (IOException e) {
+ if (e instanceof InvalidEncryptionKeyException &&
+ retry.shouldRefetchEncryptionKey()) {
+ DFSClient.LOG.info("Will fetch a new encryption key and retry, "
+ + "encryption key was invalid when connecting to " + dnInfo.addr
+ + " : " + e);
+ dfsClient.clearDataEncryptionKey();
+ retry.refetchEncryptionKey();
+ } else if (retry.shouldRefetchToken() &&
+ tokenRefetchNeeded(e, dnInfo.addr)) {
+ fetchBlockAt(block.getStartOffset());
+ retry.refetchToken();
+ } else {
+ //TODO: handles connection issues
+ DFSClient.LOG.warn("Failed to connect to " + dnInfo.addr + " for " +
+ "block" + block.getBlock(), e);
+ // re-fetch the block in case the block has been moved
+ fetchBlockAt(block.getStartOffset());
+ addToDeadNodes(dnInfo.info);
+ }
+ }
+ if (reader != null) {
+ readerInfos[chunkIndex] =
+ new BlockReaderInfo(reader, dnInfo.info, offsetInBlock);
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* Read a new stripe covering the current position, and store the data in the
* {@link #curStripeBuf}.
@@ -303,20 +294,20 @@ public class DFSStripedInputStream extends DFSInputStream {
final int stripeBufOffset = (int) (offsetInBlockGroup % stripeLen);
final int stripeLimit = (int) Math.min(currentLocatedBlock.getBlockSize()
- (stripeIndex * stripeLen), stripeLen);
- StripeRange stripeRange = new StripeRange(offsetInBlockGroup,
- stripeLimit - stripeBufOffset);
+ StripeRange stripeRange =
+ new StripeRange(offsetInBlockGroup, stripeLimit - stripeBufOffset);
LocatedStripedBlock blockGroup = (LocatedStripedBlock) currentLocatedBlock;
AlignedStripe[] stripes = StripedBlockUtil.divideOneStripe(ecPolicy,
cellSize, blockGroup, offsetInBlockGroup,
- offsetInBlockGroup + stripeRange.length - 1, curStripeBuf);
+ offsetInBlockGroup + stripeRange.getLength() - 1, curStripeBuf);
final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
blockGroup, cellSize, dataBlkNum, parityBlkNum);
// read the whole stripe
for (AlignedStripe stripe : stripes) {
// Parse group to get chosen DN location
- StripeReader sreader = new StatefulStripeReader(readingService, stripe,
- blks, blockReaders, corruptedBlocks);
+ StripeReader sreader = new StatefulStripeReader(stripe, ecPolicy, blks,
+ blockReaders, corruptedBlocks, decoder, this);
sreader.readStripe();
}
curStripeBuf.position(stripeBufOffset);
@@ -324,69 +315,8 @@ public class DFSStripedInputStream extends DFSInputStream {
curStripeRange = stripeRange;
}
- private Callable<Void> readCells(final BlockReader reader,
- final DatanodeInfo datanode, final long currentReaderOffset,
- final long targetReaderOffset, final ByteBufferStrategy[] strategies,
- final ExtendedBlock currentBlock,
- final CorruptedBlocks corruptedBlocks) {
- return new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- // reader can be null if getBlockReaderWithRetry failed or
- // the reader hit exception before
- if (reader == null) {
- throw new IOException("The BlockReader is null. " +
- "The BlockReader creation failed or the reader hit exception.");
- }
- Preconditions.checkState(currentReaderOffset <= targetReaderOffset);
- if (currentReaderOffset < targetReaderOffset) {
- long skipped = reader.skip(targetReaderOffset - currentReaderOffset);
- Preconditions.checkState(
- skipped == targetReaderOffset - currentReaderOffset);
- }
- int result = 0;
- for (ByteBufferStrategy strategy : strategies) {
- result += readToBuffer(reader, datanode, strategy, currentBlock,
- corruptedBlocks);
- }
- return null;
- }
- };
- }
-
- private int readToBuffer(BlockReader blockReader,
- DatanodeInfo currentNode, ByteBufferStrategy strategy,
- ExtendedBlock currentBlock,
- CorruptedBlocks corruptedBlocks)
- throws IOException {
- final int targetLength = strategy.getTargetLength();
- int length = 0;
- try {
- while (length < targetLength) {
- int ret = strategy.readFromBlock(blockReader);
- if (ret < 0) {
- throw new IOException("Unexpected EOS from the reader");
- }
- length += ret;
- }
- return length;
- } catch (ChecksumException ce) {
- DFSClient.LOG.warn("Found Checksum error for "
- + currentBlock + " from " + currentNode
- + " at " + ce.getPos());
- // we want to remember which block replicas we have tried
- corruptedBlocks.addCorruptedBlock(currentBlock, currentNode);
- throw ce;
- } catch (IOException e) {
- DFSClient.LOG.warn("Exception while reading from "
- + currentBlock + " of " + src + " from "
- + currentNode, e);
- throw e;
- }
- }
-
/**
- * Seek to a new arbitrary location
+ * Seek to a new arbitrary location.
*/
@Override
public synchronized void seek(long targetPos) throws IOException {
@@ -469,7 +399,7 @@ public class DFSStripedInputStream extends DFSInputStream {
}
/**
- * Copy the data from {@link #curStripeBuf} into the given buffer
+ * Copy the data from {@link #curStripeBuf} into the given buffer.
* @param strategy the ReaderStrategy containing the given buffer
* @param length target length
* @return number of bytes copied
@@ -530,17 +460,19 @@ public class DFSStripedInputStream extends DFSInputStream {
AlignedStripe[] stripes = StripedBlockUtil.divideByteRangeIntoStripes(
ecPolicy, cellSize, blockGroup, start, end, buf);
- CompletionService<Void> readService = new ExecutorCompletionService<>(
- dfsClient.getStripedReadsThreadPool());
final LocatedBlock[] blks = StripedBlockUtil.parseStripedBlockGroup(
blockGroup, cellSize, dataBlkNum, parityBlkNum);
final BlockReaderInfo[] preaderInfos = new BlockReaderInfo[groupSize];
try {
for (AlignedStripe stripe : stripes) {
// Parse group to get chosen DN location
- StripeReader preader = new PositionStripeReader(readService, stripe,
- blks, preaderInfos, corruptedBlocks);
- preader.readStripe();
+ StripeReader preader = new PositionStripeReader(stripe, ecPolicy, blks,
+ preaderInfos, corruptedBlocks, decoder, this);
+ try {
+ preader.readStripe();
+ } finally {
+ preader.close();
+ }
}
buf.position(buf.position() + (int)(end - start + 1));
} finally {
@@ -571,376 +503,6 @@ public class DFSStripedInputStream extends DFSInputStream {
}
/**
- * The reader for reading a complete {@link AlignedStripe}. Note that an
- * {@link AlignedStripe} may cross multiple stripes with cellSize width.
- */
- private abstract class StripeReader {
- final Map<Future<Void>, Integer> futures = new HashMap<>();
- final AlignedStripe alignedStripe;
- final CompletionService<Void> service;
- final LocatedBlock[] targetBlocks;
- final CorruptedBlocks corruptedBlocks;
- final BlockReaderInfo[] readerInfos;
-
- StripeReader(CompletionService<Void> service, AlignedStripe alignedStripe,
- LocatedBlock[] targetBlocks, BlockReaderInfo[] readerInfos,
- CorruptedBlocks corruptedBlocks) {
- this.service = service;
- this.alignedStripe = alignedStripe;
- this.targetBlocks = targetBlocks;
- this.readerInfos = readerInfos;
- this.corruptedBlocks = corruptedBlocks;
- }
-
- /** prepare all the data chunks */
- abstract void prepareDecodeInputs();
-
- /** prepare the parity chunk and block reader if necessary */
- abstract boolean prepareParityChunk(int index);
-
- abstract void decode();
-
- void updateState4SuccessRead(StripingChunkReadResult result) {
- Preconditions.checkArgument(
- result.state == StripingChunkReadResult.SUCCESSFUL);
- readerInfos[result.index].setOffset(alignedStripe.getOffsetInBlock()
- + alignedStripe.getSpanInBlock());
- }
-
- private void checkMissingBlocks() throws IOException {
- if (alignedStripe.missingChunksNum > parityBlkNum) {
- clearFutures(futures.keySet());
- throw new IOException(alignedStripe.missingChunksNum
- + " missing blocks, the stripe is: " + alignedStripe
- + "; locatedBlocks is: " + locatedBlocks);
- }
- }
-
- /**
- * We need decoding. Thus go through all the data chunks and make sure we
- * submit read requests for all of them.
- */
- private void readDataForDecoding() throws IOException {
- prepareDecodeInputs();
- for (int i = 0; i < dataBlkNum; i++) {
- Preconditions.checkNotNull(alignedStripe.chunks[i]);
- if (alignedStripe.chunks[i].state == StripingChunk.REQUESTED) {
- if (!readChunk(targetBlocks[i], i)) {
- alignedStripe.missingChunksNum++;
- }
- }
- }
- checkMissingBlocks();
- }
-
- void readParityChunks(int num) throws IOException {
- for (int i = dataBlkNum, j = 0; i < dataBlkNum + parityBlkNum && j < num;
- i++) {
- if (alignedStripe.chunks[i] == null) {
- if (prepareParityChunk(i) && readChunk(targetBlocks[i], i)) {
- j++;
- } else {
- alignedStripe.missingChunksNum++;
- }
- }
- }
- checkMissingBlocks();
- }
-
- boolean createBlockReader(LocatedBlock block, int chunkIndex)
- throws IOException {
- BlockReader reader = null;
- final ReaderRetryPolicy retry = new ReaderRetryPolicy();
- DNAddrPair dnInfo = new DNAddrPair(null, null, null);
-
- while(true) {
- try {
- // the cached block location might have been re-fetched, so always
- // get it from cache.
- block = refreshLocatedBlock(block);
- targetBlocks[chunkIndex] = block;
-
- // internal block has one location, just rule out the deadNodes
- dnInfo = getBestNodeDNAddrPair(block, null);
- if (dnInfo == null) {
- break;
- }
- reader = getBlockReader(block, alignedStripe.getOffsetInBlock(),
- block.getBlockSize() - alignedStripe.getOffsetInBlock(),
- dnInfo.addr, dnInfo.storageType, dnInfo.info);
- } catch (IOException e) {
- if (e instanceof InvalidEncryptionKeyException &&
- retry.shouldRefetchEncryptionKey()) {
- DFSClient.LOG.info("Will fetch a new encryption key and retry, "
- + "encryption key was invalid when connecting to " + dnInfo.addr
- + " : " + e);
- dfsClient.clearDataEncryptionKey();
- retry.refetchEncryptionKey();
- } else if (retry.shouldRefetchToken() &&
- tokenRefetchNeeded(e, dnInfo.addr)) {
- fetchBlockAt(block.getStartOffset());
- retry.refetchToken();
- } else {
- //TODO: handles connection issues
- DFSClient.LOG.warn("Failed to connect to " + dnInfo.addr + " for " +
- "block" + block.getBlock(), e);
- // re-fetch the block in case the block has been moved
- fetchBlockAt(block.getStartOffset());
- addToDeadNodes(dnInfo.info);
- }
- }
- if (reader != null) {
- readerInfos[chunkIndex] = new BlockReaderInfo(reader, dnInfo.info,
- alignedStripe.getOffsetInBlock());
- return true;
- }
- }
- return false;
- }
-
- private ByteBufferStrategy[] getReadStrategies(StripingChunk chunk) {
- if (chunk.useByteBuffer()) {
- ByteBufferStrategy strategy = new ByteBufferStrategy(
- chunk.getByteBuffer(), readStatistics, dfsClient);
- return new ByteBufferStrategy[]{strategy};
- } else {
- ByteBufferStrategy[] strategies =
- new ByteBufferStrategy[chunk.getChunkBuffer().getSlices().size()];
- for (int i = 0; i < strategies.length; i++) {
- ByteBuffer buffer = chunk.getChunkBuffer().getSlice(i);
- strategies[i] =
- new ByteBufferStrategy(buffer, readStatistics, dfsClient);
- }
- return strategies;
- }
- }
-
- boolean readChunk(final LocatedBlock block, int chunkIndex)
- throws IOException {
- final StripingChunk chunk = alignedStripe.chunks[chunkIndex];
- if (block == null) {
- chunk.state = StripingChunk.MISSING;
- return false;
- }
- if (readerInfos[chunkIndex] == null) {
- if (!createBlockReader(block, chunkIndex)) {
- chunk.state = StripingChunk.MISSING;
- return false;
- }
- } else if (readerInfos[chunkIndex].shouldSkip) {
- chunk.state = StripingChunk.MISSING;
- return false;
- }
-
- chunk.state = StripingChunk.PENDING;
- Callable<Void> readCallable = readCells(readerInfos[chunkIndex].reader,
- readerInfos[chunkIndex].datanode,
- readerInfos[chunkIndex].blockReaderOffset,
- alignedStripe.getOffsetInBlock(), getReadStrategies(chunk),
- block.getBlock(), corruptedBlocks);
-
- Future<Void> request = service.submit(readCallable);
- futures.put(request, chunkIndex);
- return true;
- }
-
- /** read the whole stripe. do decoding if necessary */
- void readStripe() throws IOException {
- for (int i = 0; i < dataBlkNum; i++) {
- if (alignedStripe.chunks[i] != null &&
- alignedStripe.chunks[i].state != StripingChunk.ALLZERO) {
- if (!readChunk(targetBlocks[i], i)) {
- alignedStripe.missingChunksNum++;
- }
- }
- }
- // There are missing block locations at this stage. Thus we need to read
- // the full stripe and one more parity block.
- if (alignedStripe.missingChunksNum > 0) {
- checkMissingBlocks();
- readDataForDecoding();
- // read parity chunks
- readParityChunks(alignedStripe.missingChunksNum);
- }
- // TODO: for a full stripe we can start reading (dataBlkNum + 1) chunks
-
- // Input buffers for potential decode operation, which remains null until
- // first read failure
- while (!futures.isEmpty()) {
- try {
- StripingChunkReadResult r = StripedBlockUtil
- .getNextCompletedStripedRead(service, futures, 0);
- if (DFSClient.LOG.isDebugEnabled()) {
- DFSClient.LOG.debug("Read task returned: " + r + ", for stripe "
- + alignedStripe);
- }
- StripingChunk returnedChunk = alignedStripe.chunks[r.index];
- Preconditions.checkNotNull(returnedChunk);
- Preconditions.checkState(returnedChunk.state == StripingChunk.PENDING);
-
- if (r.state == StripingChunkReadResult.SUCCESSFUL) {
- returnedChunk.state = StripingChunk.FETCHED;
- alignedStripe.fetchedChunksNum++;
- updateState4SuccessRead(r);
- if (alignedStripe.fetchedChunksNum == dataBlkNum) {
- clearFutures(futures.keySet());
- break;
- }
- } else {
- returnedChunk.state = StripingChunk.MISSING;
- // close the corresponding reader
- closeReader(readerInfos[r.index]);
-
- final int missing = alignedStripe.missingChunksNum;
- alignedStripe.missingChunksNum++;
- checkMissingBlocks();
-
- readDataForDecoding();
- readParityChunks(alignedStripe.missingChunksNum - missing);
- }
- } catch (InterruptedException ie) {
- String err = "Read request interrupted";
- DFSClient.LOG.error(err);
- clearFutures(futures.keySet());
- // Don't decode if read interrupted
- throw new InterruptedIOException(err);
- }
- }
-
- if (alignedStripe.missingChunksNum > 0) {
- decode();
- }
- }
- }
-
- class PositionStripeReader extends StripeReader {
- private ByteBuffer[] decodeInputs = null;
-
- PositionStripeReader(CompletionService<Void> service,
- AlignedStripe alignedStripe, LocatedBlock[] targetBlocks,
- BlockReaderInfo[] readerInfos, CorruptedBlocks corruptedBlocks) {
- super(service, alignedStripe, targetBlocks, readerInfos,
- corruptedBlocks);
- }
-
- @Override
- void prepareDecodeInputs() {
- if (decodeInputs == null) {
- decodeInputs = StripedBlockUtil.initDecodeInputs(alignedStripe,
- dataBlkNum, parityBlkNum);
- }
- }
-
- @Override
- boolean prepareParityChunk(int index) {
- Preconditions.checkState(index >= dataBlkNum &&
- alignedStripe.chunks[index] == null);
- alignedStripe.chunks[index] = new StripingChunk(decodeInputs[index]);
- return true;
- }
-
- @Override
- void decode() {
- StripedBlockUtil.finalizeDecodeInputs(decodeInputs, alignedStripe);
- StripedBlockUtil.decodeAndFillBuffer(decodeInputs, alignedStripe,
- dataBlkNum, parityBlkNum, decoder);
- }
- }
-
- class StatefulStripeReader extends StripeReader {
- ByteBuffer[] decodeInputs;
-
- StatefulStripeReader(CompletionService<Void> service,
- AlignedStripe alignedStripe, LocatedBlock[] targetBlocks,
- BlockReaderInfo[] readerInfos, CorruptedBlocks corruptedBlocks) {
- super(service, alignedStripe, targetBlocks, readerInfos,
- corruptedBlocks);
- }
-
- @Override
- void prepareDecodeInputs() {
- if (decodeInputs == null) {
- decodeInputs = new ByteBuffer[dataBlkNum + parityBlkNum];
- final ByteBuffer cur;
- synchronized (DFSStripedInputStream.this) {
- cur = curStripeBuf.duplicate();
- }
- StripedBlockUtil.VerticalRange range = alignedStripe.range;
- for (int i = 0; i < dataBlkNum; i++) {
- cur.limit(cur.capacity());
- int pos = (int) (range.offsetInBlock % cellSize + cellSize * i);
- cur.position(pos);
- cur.limit((int) (pos + range.spanInBlock));
- decodeInputs[i] = cur.slice();
- if (alignedStripe.chunks[i] == null) {
- alignedStripe.chunks[i] = new StripingChunk(decodeInputs[i]);
- }
- }
- }
- }
-
- @Override
- boolean prepareParityChunk(int index) {
- Preconditions.checkState(index >= dataBlkNum
- && alignedStripe.chunks[index] == null);
- if (blockReaders[index] != null && blockReaders[index].shouldSkip) {
- alignedStripe.chunks[index] = new StripingChunk(StripingChunk.MISSING);
- // we have failed the block reader before
- return false;
- }
- final int parityIndex = index - dataBlkNum;
- ByteBuffer buf = getParityBuffer().duplicate();
- buf.position(cellSize * parityIndex);
- buf.limit(cellSize * parityIndex + (int) alignedStripe.range.spanInBlock);
- decodeInputs[index] = buf.slice();
- alignedStripe.chunks[index] = new StripingChunk(decodeInputs[index]);
- return true;
- }
-
- @Override
- void decode() {
- final int span = (int) alignedStripe.getSpanInBlock();
- for (int i = 0; i < alignedStripe.chunks.length; i++) {
- if (alignedStripe.chunks[i] != null &&
- alignedStripe.chunks[i].state == StripingChunk.ALLZERO) {
- for (int j = 0; j < span; j++) {
- decodeInputs[i].put((byte) 0);
- }
- decodeInputs[i].flip();
- } else if (alignedStripe.chunks[i] != null &&
- alignedStripe.chunks[i].state == StripingChunk.FETCHED) {
- decodeInputs[i].position(0);
- decodeInputs[i].limit(span);
- }
- }
- int[] decodeIndices = new int[parityBlkNum];
- int pos = 0;
- for (int i = 0; i < alignedStripe.chunks.length; i++) {
- if (alignedStripe.chunks[i] != null &&
- alignedStripe.chunks[i].state == StripingChunk.MISSING) {
- if (i < dataBlkNum) {
- decodeIndices[pos++] = i;
- } else {
- decodeInputs[i] = null;
- }
- }
- }
- decodeIndices = Arrays.copyOf(decodeIndices, pos);
-
- final int decodeChunkNum = decodeIndices.length;
- ByteBuffer[] outputs = new ByteBuffer[decodeChunkNum];
- for (int i = 0; i < decodeChunkNum; i++) {
- outputs[i] = decodeInputs[decodeIndices[i]];
- outputs[i].position(0);
- outputs[i].limit((int) alignedStripe.range.spanInBlock);
- decodeInputs[decodeIndices[i]] = null;
- }
-
- decoder.decode(decodeInputs, decodeIndices, outputs);
- }
- }
-
- /**
* May need online read recovery, zero-copy read doesn't make
* sense, so don't support it.
*/
@@ -957,12 +519,4 @@ public class DFSStripedInputStream extends DFSInputStream {
throw new UnsupportedOperationException(
"Not support enhanced byte buffer access.");
}
-
- /** A variation to {@link DFSInputStream#cancelAll} */
- private void clearFutures(Collection<Future<Void>> futures) {
- for (Future<Void> future : futures) {
- future.cancel(false);
- }
- futures.clear();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
new file mode 100644
index 0000000..5818291
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/PositionStripeReader.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.configuration.SystemConfiguration;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunk;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
+
+import java.nio.ByteBuffer;
+
+/**
+ * The reader for reading a complete {@link StripedBlockUtil.AlignedStripe}
+ * which may cross multiple stripes with cellSize width.
+ */
+class PositionStripeReader extends StripeReader {
+ private ByteBuffer codingBuffer;
+
+ PositionStripeReader(AlignedStripe alignedStripe,
+ ErasureCodingPolicy ecPolicy, LocatedBlock[] targetBlocks,
+ BlockReaderInfo[] readerInfos, CorruptedBlocks corruptedBlocks,
+ RawErasureDecoder decoder, DFSStripedInputStream dfsStripedInputStream) {
+ super(alignedStripe, ecPolicy, targetBlocks, readerInfos,
+ corruptedBlocks, decoder, dfsStripedInputStream);
+ }
+
+ @Override
+ void prepareDecodeInputs() {
+ if (codingBuffer == null) {
+ this.decodeInputs = new ECChunk[dataBlkNum + parityBlkNum];
+ initDecodeInputs(alignedStripe);
+ }
+ }
+
+ @Override
+ boolean prepareParityChunk(int index) {
+ Preconditions.checkState(index >= dataBlkNum &&
+ alignedStripe.chunks[index] == null);
+
+ alignedStripe.chunks[index] =
+ new StripingChunk(decodeInputs[index].getBuffer());
+
+ return true;
+ }
+
+ @Override
+ void decode() {
+ finalizeDecodeInputs();
+ decodeAndFillBuffer(true);
+ }
+
+ void initDecodeInputs(AlignedStripe alignedStripe) {
+ int bufLen = (int) alignedStripe.getSpanInBlock();
+ int bufCount = dataBlkNum + parityBlkNum;
+ codingBuffer = dfsStripedInputStream.getBufferPool().
+ getBuffer(useDirectBuffer(), bufLen * bufCount);
+ ByteBuffer buffer;
+ for (int i = 0; i < decodeInputs.length; i++) {
+ buffer = codingBuffer.duplicate();
+ decodeInputs[i] = new ECChunk(buffer, i * bufLen, bufLen);
+ }
+
+ for (int i = 0; i < dataBlkNum; i++) {
+ if (alignedStripe.chunks[i] == null) {
+ alignedStripe.chunks[i] =
+ new StripingChunk(decodeInputs[i].getBuffer());
+ }
+ }
+ }
+
+ void close() {
+ if (decodeInputs != null) {
+ for (int i = 0; i < decodeInputs.length; i++) {
+ decodeInputs[i] = null;
+ }
+ }
+
+ if (codingBuffer != null) {
+ dfsStripedInputStream.getBufferPool().putBuffer(codingBuffer);
+ codingBuffer = null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
new file mode 100644
index 0000000..8879514
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StatefulStripeReader.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunk;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
+
+import java.nio.ByteBuffer;
+
+/**
+ * The reader for reading a complete {@link StripedBlockUtil.AlignedStripe}
+ * which belongs to a single stripe.
+ * Reading cross multiple strips is not supported in this reader.
+ */
+class StatefulStripeReader extends StripeReader {
+
+ StatefulStripeReader(AlignedStripe alignedStripe,
+ ErasureCodingPolicy ecPolicy, LocatedBlock[] targetBlocks,
+ BlockReaderInfo[] readerInfos, CorruptedBlocks corruptedBlocks,
+ RawErasureDecoder decoder, DFSStripedInputStream dfsStripedInputStream) {
+ super(alignedStripe, ecPolicy, targetBlocks, readerInfos,
+ corruptedBlocks, decoder, dfsStripedInputStream);
+ }
+
+ @Override
+ void prepareDecodeInputs() {
+ final ByteBuffer cur;
+ synchronized (dfsStripedInputStream) {
+ cur = dfsStripedInputStream.getCurStripeBuf().duplicate();
+ }
+
+ this.decodeInputs = new ECChunk[dataBlkNum + parityBlkNum];
+ int bufLen = (int) alignedStripe.getSpanInBlock();
+ int bufOff = (int) alignedStripe.getOffsetInBlock();
+ for (int i = 0; i < dataBlkNum; i++) {
+ cur.limit(cur.capacity());
+ int pos = bufOff % cellSize + cellSize * i;
+ cur.position(pos);
+ cur.limit(pos + bufLen);
+ decodeInputs[i] = new ECChunk(cur.slice(), 0, bufLen);
+ if (alignedStripe.chunks[i] == null) {
+ alignedStripe.chunks[i] =
+ new StripingChunk(decodeInputs[i].getBuffer());
+ }
+ }
+ }
+
+ @Override
+ boolean prepareParityChunk(int index) {
+ Preconditions.checkState(index >= dataBlkNum
+ && alignedStripe.chunks[index] == null);
+ if (readerInfos[index] != null && readerInfos[index].shouldSkip) {
+ alignedStripe.chunks[index] = new StripingChunk(StripingChunk.MISSING);
+ // we have failed the block reader before
+ return false;
+ }
+ final int parityIndex = index - dataBlkNum;
+ ByteBuffer buf = dfsStripedInputStream.getParityBuffer().duplicate();
+ buf.position(cellSize * parityIndex);
+ buf.limit(cellSize * parityIndex + (int) alignedStripe.range.spanInBlock);
+ decodeInputs[index] =
+ new ECChunk(buf.slice(), 0, (int) alignedStripe.range.spanInBlock);
+ alignedStripe.chunks[index] =
+ new StripingChunk(decodeInputs[index].getBuffer());
+ return true;
+ }
+
+ @Override
+ void decode() {
+ finalizeDecodeInputs();
+ decodeAndFillBuffer(false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
new file mode 100644
index 0000000..5518752
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/StripeReader.java
@@ -0,0 +1,463 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.fs.ChecksumException;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunk;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.AlignedStripe;
+import org.apache.hadoop.hdfs.util.StripedBlockUtil.StripingChunkReadResult;
+import org.apache.hadoop.io.erasurecode.ECChunk;
+import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.hdfs.DFSUtilClient.CorruptedBlocks;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.nio.ByteBuffer;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.Future;
+
+/**
+ * The reader for reading a complete {@link StripedBlockUtil.AlignedStripe}.
+ * Note that an {@link StripedBlockUtil.AlignedStripe} may cross multiple
+ * stripes with cellSize width.
+ */
+abstract class StripeReader {
+
+ static class ReaderRetryPolicy {
+ private int fetchEncryptionKeyTimes = 1;
+ private int fetchTokenTimes = 1;
+
+ void refetchEncryptionKey() {
+ fetchEncryptionKeyTimes--;
+ }
+
+ void refetchToken() {
+ fetchTokenTimes--;
+ }
+
+ boolean shouldRefetchEncryptionKey() {
+ return fetchEncryptionKeyTimes > 0;
+ }
+
+ boolean shouldRefetchToken() {
+ return fetchTokenTimes > 0;
+ }
+ }
+
+ static class BlockReaderInfo {
+ final BlockReader reader;
+ final DatanodeInfo datanode;
+ /**
+ * when initializing block readers, their starting offsets are set to the
+ * same number: the smallest internal block offsets among all the readers.
+ * This is because it is possible that for some internal blocks we have to
+ * read "backwards" for decoding purpose. We thus use this offset array to
+ * track offsets for all the block readers so that we can skip data if
+ * necessary.
+ */
+ long blockReaderOffset;
+ /**
+ * We use this field to indicate whether we should use this reader. In case
+ * we hit any issue with this reader, we set this field to true and avoid
+ * using it for the next stripe.
+ */
+ boolean shouldSkip = false;
+
+ BlockReaderInfo(BlockReader reader, DatanodeInfo dn, long offset) {
+ this.reader = reader;
+ this.datanode = dn;
+ this.blockReaderOffset = offset;
+ }
+
+ void setOffset(long offset) {
+ this.blockReaderOffset = offset;
+ }
+
+ void skip() {
+ this.shouldSkip = true;
+ }
+ }
+
+ protected final Map<Future<Void>, Integer> futures = new HashMap<>();
+ protected final AlignedStripe alignedStripe;
+ protected final CompletionService<Void> service;
+ protected final LocatedBlock[] targetBlocks;
+ protected final CorruptedBlocks corruptedBlocks;
+ protected final BlockReaderInfo[] readerInfos;
+ protected final ErasureCodingPolicy ecPolicy;
+ protected final short dataBlkNum;
+ protected final short parityBlkNum;
+ protected final int cellSize;
+ protected final RawErasureDecoder decoder;
+ protected final DFSStripedInputStream dfsStripedInputStream;
+
+ protected ECChunk[] decodeInputs;
+
+ StripeReader(AlignedStripe alignedStripe,
+ ErasureCodingPolicy ecPolicy, LocatedBlock[] targetBlocks,
+ BlockReaderInfo[] readerInfos, CorruptedBlocks corruptedBlocks,
+ RawErasureDecoder decoder,
+ DFSStripedInputStream dfsStripedInputStream) {
+ this.alignedStripe = alignedStripe;
+ this.ecPolicy = ecPolicy;
+ this.dataBlkNum = (short)ecPolicy.getNumDataUnits();
+ this.parityBlkNum = (short)ecPolicy.getNumParityUnits();
+ this.cellSize = ecPolicy.getCellSize();
+ this.targetBlocks = targetBlocks;
+ this.readerInfos = readerInfos;
+ this.corruptedBlocks = corruptedBlocks;
+ this.decoder = decoder;
+ this.dfsStripedInputStream = dfsStripedInputStream;
+
+ service = new ExecutorCompletionService<>(
+ dfsStripedInputStream.getStripedReadsThreadPool());
+ }
+
+ /**
+ * Prepare all the data chunks.
+ */
+ abstract void prepareDecodeInputs();
+
+ /**
+ * Prepare the parity chunk and block reader if necessary.
+ */
+ abstract boolean prepareParityChunk(int index);
+
+ /*
+ * Decode to get the missing data.
+ */
+ abstract void decode();
+
+ /*
+ * Default close do nothing.
+ */
+ void close() {
+ }
+
+ void updateState4SuccessRead(StripingChunkReadResult result) {
+ Preconditions.checkArgument(
+ result.state == StripingChunkReadResult.SUCCESSFUL);
+ readerInfos[result.index].setOffset(alignedStripe.getOffsetInBlock()
+ + alignedStripe.getSpanInBlock());
+ }
+
+ private void checkMissingBlocks() throws IOException {
+ if (alignedStripe.missingChunksNum > parityBlkNum) {
+ clearFutures();
+ throw new IOException(alignedStripe.missingChunksNum
+ + " missing blocks, the stripe is: " + alignedStripe
+ + "; locatedBlocks is: " + dfsStripedInputStream.getLocatedBlocks());
+ }
+ }
+
+ /**
+ * We need decoding. Thus go through all the data chunks and make sure we
+ * submit read requests for all of them.
+ */
+ private void readDataForDecoding() throws IOException {
+ prepareDecodeInputs();
+ for (int i = 0; i < dataBlkNum; i++) {
+ Preconditions.checkNotNull(alignedStripe.chunks[i]);
+ if (alignedStripe.chunks[i].state == StripingChunk.REQUESTED) {
+ if (!readChunk(targetBlocks[i], i)) {
+ alignedStripe.missingChunksNum++;
+ }
+ }
+ }
+ checkMissingBlocks();
+ }
+
+ void readParityChunks(int num) throws IOException {
+ for (int i = dataBlkNum, j = 0; i < dataBlkNum + parityBlkNum && j < num;
+ i++) {
+ if (alignedStripe.chunks[i] == null) {
+ if (prepareParityChunk(i) && readChunk(targetBlocks[i], i)) {
+ j++;
+ } else {
+ alignedStripe.missingChunksNum++;
+ }
+ }
+ }
+ checkMissingBlocks();
+ }
+
+ private ByteBufferStrategy[] getReadStrategies(StripingChunk chunk) {
+ if (chunk.useByteBuffer()) {
+ ByteBufferStrategy strategy = new ByteBufferStrategy(
+ chunk.getByteBuffer(), dfsStripedInputStream.getReadStatistics(),
+ dfsStripedInputStream.getDFSClient());
+ return new ByteBufferStrategy[]{strategy};
+ }
+
+ ByteBufferStrategy[] strategies =
+ new ByteBufferStrategy[chunk.getChunkBuffer().getSlices().size()];
+ for (int i = 0; i < strategies.length; i++) {
+ ByteBuffer buffer = chunk.getChunkBuffer().getSlice(i);
+ strategies[i] = new ByteBufferStrategy(buffer,
+ dfsStripedInputStream.getReadStatistics(),
+ dfsStripedInputStream.getDFSClient());
+ }
+ return strategies;
+ }
+
+ private int readToBuffer(BlockReader blockReader,
+ DatanodeInfo currentNode, ByteBufferStrategy strategy,
+ ExtendedBlock currentBlock) throws IOException {
+ final int targetLength = strategy.getTargetLength();
+ int length = 0;
+ try {
+ while (length < targetLength) {
+ int ret = strategy.readFromBlock(blockReader);
+ if (ret < 0) {
+ throw new IOException("Unexpected EOS from the reader");
+ }
+ length += ret;
+ }
+ return length;
+ } catch (ChecksumException ce) {
+ DFSClient.LOG.warn("Found Checksum error for "
+ + currentBlock + " from " + currentNode
+ + " at " + ce.getPos());
+ // we want to remember which block replicas we have tried
+ corruptedBlocks.addCorruptedBlock(currentBlock, currentNode);
+ throw ce;
+ } catch (IOException e) {
+ DFSClient.LOG.warn("Exception while reading from "
+ + currentBlock + " of " + dfsStripedInputStream.getSrc() + " from "
+ + currentNode, e);
+ throw e;
+ }
+ }
+
+ private Callable<Void> readCells(final BlockReader reader,
+ final DatanodeInfo datanode, final long currentReaderOffset,
+ final long targetReaderOffset, final ByteBufferStrategy[] strategies,
+ final ExtendedBlock currentBlock) {
+ return () -> {
+ // reader can be null if getBlockReaderWithRetry failed or
+ // the reader hit exception before
+ if (reader == null) {
+ throw new IOException("The BlockReader is null. " +
+ "The BlockReader creation failed or the reader hit exception.");
+ }
+ Preconditions.checkState(currentReaderOffset <= targetReaderOffset);
+ if (currentReaderOffset < targetReaderOffset) {
+ long skipped = reader.skip(targetReaderOffset - currentReaderOffset);
+ Preconditions.checkState(
+ skipped == targetReaderOffset - currentReaderOffset);
+ }
+
+ for (ByteBufferStrategy strategy : strategies) {
+ readToBuffer(reader, datanode, strategy, currentBlock);
+ }
+ return null;
+ };
+ }
+
+ boolean readChunk(final LocatedBlock block, int chunkIndex)
+ throws IOException {
+ final StripingChunk chunk = alignedStripe.chunks[chunkIndex];
+ if (block == null) {
+ chunk.state = StripingChunk.MISSING;
+ return false;
+ }
+
+ if (readerInfos[chunkIndex] == null) {
+ if (!dfsStripedInputStream.createBlockReader(block,
+ alignedStripe.getOffsetInBlock(), targetBlocks,
+ readerInfos, chunkIndex)) {
+ chunk.state = StripingChunk.MISSING;
+ return false;
+ }
+ } else if (readerInfos[chunkIndex].shouldSkip) {
+ chunk.state = StripingChunk.MISSING;
+ return false;
+ }
+
+ chunk.state = StripingChunk.PENDING;
+ Callable<Void> readCallable = readCells(readerInfos[chunkIndex].reader,
+ readerInfos[chunkIndex].datanode,
+ readerInfos[chunkIndex].blockReaderOffset,
+ alignedStripe.getOffsetInBlock(), getReadStrategies(chunk),
+ block.getBlock());
+
+ Future<Void> request = service.submit(readCallable);
+ futures.put(request, chunkIndex);
+ return true;
+ }
+
+ /**
+ * read the whole stripe. do decoding if necessary
+ */
+ void readStripe() throws IOException {
+ for (int i = 0; i < dataBlkNum; i++) {
+ if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state != StripingChunk.ALLZERO) {
+ if (!readChunk(targetBlocks[i], i)) {
+ alignedStripe.missingChunksNum++;
+ }
+ }
+ }
+ // There are missing block locations at this stage. Thus we need to read
+ // the full stripe and one more parity block.
+ if (alignedStripe.missingChunksNum > 0) {
+ checkMissingBlocks();
+ readDataForDecoding();
+ // read parity chunks
+ readParityChunks(alignedStripe.missingChunksNum);
+ }
+ // TODO: for a full stripe we can start reading (dataBlkNum + 1) chunks
+
+ // Input buffers for potential decode operation, which remains null until
+ // first read failure
+ while (!futures.isEmpty()) {
+ try {
+ StripingChunkReadResult r = StripedBlockUtil
+ .getNextCompletedStripedRead(service, futures, 0);
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug("Read task returned: " + r + ", for stripe "
+ + alignedStripe);
+ }
+ StripingChunk returnedChunk = alignedStripe.chunks[r.index];
+ Preconditions.checkNotNull(returnedChunk);
+ Preconditions.checkState(returnedChunk.state == StripingChunk.PENDING);
+
+ if (r.state == StripingChunkReadResult.SUCCESSFUL) {
+ returnedChunk.state = StripingChunk.FETCHED;
+ alignedStripe.fetchedChunksNum++;
+ updateState4SuccessRead(r);
+ if (alignedStripe.fetchedChunksNum == dataBlkNum) {
+ clearFutures();
+ break;
+ }
+ } else {
+ returnedChunk.state = StripingChunk.MISSING;
+ // close the corresponding reader
+ dfsStripedInputStream.closeReader(readerInfos[r.index]);
+
+ final int missing = alignedStripe.missingChunksNum;
+ alignedStripe.missingChunksNum++;
+ checkMissingBlocks();
+
+ readDataForDecoding();
+ readParityChunks(alignedStripe.missingChunksNum - missing);
+ }
+ } catch (InterruptedException ie) {
+ String err = "Read request interrupted";
+ DFSClient.LOG.error(err);
+ clearFutures();
+ // Don't decode if read interrupted
+ throw new InterruptedIOException(err);
+ }
+ }
+
+ if (alignedStripe.missingChunksNum > 0) {
+ decode();
+ }
+ }
+
+ /**
+ * Some fetched {@link StripingChunk} might be stored in original application
+ * buffer instead of prepared decode input buffers. Some others are beyond
+ * the range of the internal blocks and should correspond to all zero bytes.
+ * When all pending requests have returned, this method should be called to
+ * finalize decode input buffers.
+ */
+
+ void finalizeDecodeInputs() {
+ for (int i = 0; i < alignedStripe.chunks.length; i++) {
+ final StripingChunk chunk = alignedStripe.chunks[i];
+ if (chunk != null && chunk.state == StripingChunk.FETCHED) {
+ if (chunk.useChunkBuffer()) {
+ chunk.getChunkBuffer().copyTo(decodeInputs[i].getBuffer());
+ } else {
+ chunk.getByteBuffer().flip();
+ }
+ } else if (chunk != null && chunk.state == StripingChunk.ALLZERO) {
+ decodeInputs[i].setAllZero(true);
+ }
+ }
+ }
+
+ /**
+ * Decode based on the given input buffers and erasure coding policy.
+ */
+ void decodeAndFillBuffer(boolean fillBuffer) {
+ // Step 1: prepare indices and output buffers for missing data units
+ int[] decodeIndices = prepareErasedIndices();
+
+ final int decodeChunkNum = decodeIndices.length;
+ ECChunk[] outputs = new ECChunk[decodeChunkNum];
+ for (int i = 0; i < decodeChunkNum; i++) {
+ outputs[i] = decodeInputs[decodeIndices[i]];
+ decodeInputs[decodeIndices[i]] = null;
+ }
+ // Step 2: decode into prepared output buffers
+ decoder.decode(decodeInputs, decodeIndices, outputs);
+
+ // Step 3: fill original application buffer with decoded data
+ if (fillBuffer) {
+ for (int i = 0; i < decodeIndices.length; i++) {
+ int missingBlkIdx = decodeIndices[i];
+ StripingChunk chunk = alignedStripe.chunks[missingBlkIdx];
+ if (chunk.state == StripingChunk.MISSING && chunk.useChunkBuffer()) {
+ chunk.getChunkBuffer().copyFrom(outputs[i].getBuffer());
+ }
+ }
+ }
+ }
+
+ /**
+ * Prepare erased indices.
+ */
+ int[] prepareErasedIndices() {
+ int[] decodeIndices = new int[parityBlkNum];
+ int pos = 0;
+ for (int i = 0; i < alignedStripe.chunks.length; i++) {
+ if (alignedStripe.chunks[i] != null &&
+ alignedStripe.chunks[i].state == StripingChunk.MISSING){
+ decodeIndices[pos++] = i;
+ }
+ }
+
+ int[] erasedIndices = Arrays.copyOf(decodeIndices, pos);
+ return erasedIndices;
+ }
+
+ void clearFutures() {
+ for (Future<Void> future : futures.keySet()) {
+ future.cancel(false);
+ }
+ futures.clear();
+ }
+
+ boolean useDirectBuffer() {
+ return decoder.preferDirectBuffer();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
index 4dbbc3d..896ebc6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/util/StripedBlockUtil.java
@@ -22,7 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.DFSStripedOutputStream;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
@@ -32,7 +31,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import com.google.common.base.Preconditions;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
-import org.apache.hadoop.io.erasurecode.rawcoder.RawErasureDecoder;
+import org.apache.hadoop.hdfs.DFSStripedOutputStream;
import org.apache.hadoop.security.token.Token;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -77,18 +76,6 @@ public class StripedBlockUtil {
LoggerFactory.getLogger(StripedBlockUtil.class);
/**
- * Parses a striped block group into individual blocks.
- * @param bg The striped block group
- * @param ecPolicy The erasure coding policy
- * @return An array of the blocks in the group
- */
- public static LocatedBlock[] parseStripedBlockGroup(LocatedStripedBlock bg,
- ErasureCodingPolicy ecPolicy) {
- return parseStripedBlockGroup(bg, ecPolicy.getCellSize(),
- ecPolicy.getNumDataUnits(), ecPolicy.getNumParityUnits());
- }
-
- /**
* This method parses a striped block group into individual blocks.
*
* @param bg The striped block group
@@ -112,7 +99,7 @@ public class StripedBlockUtil {
}
/**
- * This method creates an internal block at the given index of a block group
+ * This method creates an internal block at the given index of a block group.
*
* @param idxInReturnedLocs The index in the stored locations in the
* {@link LocatedStripedBlock} object
@@ -169,7 +156,7 @@ public class StripedBlockUtil {
}
/**
- * Get the size of an internal block at the given index of a block group
+ * Get the size of an internal block at the given index of a block group.
*
* @param dataSize Size of the block group only counting data blocks
* @param cellSize The size of a striping cell
@@ -237,7 +224,7 @@ public class StripedBlockUtil {
/**
* Given a byte's offset in an internal block, calculate the offset in
- * the block group
+ * the block group.
*/
public static long offsetInBlkToOffsetInBG(int cellSize, int dataBlkNum,
long offsetInBlk, int idxInBlockGroup) {
@@ -248,12 +235,12 @@ public class StripedBlockUtil {
}
/**
- * Get the next completed striped read task
+ * Get the next completed striped read task.
*
- * @return {@link StripingChunkReadResult} indicating the status of the read task
- * succeeded, and the block index of the task. If the method times
- * out without getting any completed read tasks, -1 is returned as
- * block index.
+ * @return {@link StripingChunkReadResult} indicating the status of the read
+ * task succeeded, and the block index of the task. If the method
+ * times out without getting any completed read tasks, -1 is
+ * returned as block index.
* @throws InterruptedException
*/
public static StripingChunkReadResult getNextCompletedStripedRead(
@@ -287,7 +274,7 @@ public class StripedBlockUtil {
/**
* Get the total usage of the striped blocks, which is the total of data
- * blocks and parity blocks
+ * blocks and parity blocks.
*
* @param numDataBlkBytes
* Size of the block group only counting data blocks
@@ -308,91 +295,6 @@ public class StripedBlockUtil {
}
/**
- * Initialize the decoding input buffers based on the chunk states in an
- * {@link AlignedStripe}. For each chunk that was not initially requested,
- * schedule a new fetch request with the decoding input buffer as transfer
- * destination.
- */
- public static ByteBuffer[] initDecodeInputs(AlignedStripe alignedStripe,
- int dataBlkNum, int parityBlkNum) {
- ByteBuffer[] decodeInputs = new ByteBuffer[dataBlkNum + parityBlkNum];
- for (int i = 0; i < decodeInputs.length; i++) {
- decodeInputs[i] = ByteBuffer.allocate(
- (int) alignedStripe.getSpanInBlock());
- }
- // read the full data aligned stripe
- for (int i = 0; i < dataBlkNum; i++) {
- if (alignedStripe.chunks[i] == null) {
- alignedStripe.chunks[i] = new StripingChunk(decodeInputs[i]);
- }
- }
- return decodeInputs;
- }
-
- /**
- * Some fetched {@link StripingChunk} might be stored in original application
- * buffer instead of prepared decode input buffers. Some others are beyond
- * the range of the internal blocks and should correspond to all zero bytes.
- * When all pending requests have returned, this method should be called to
- * finalize decode input buffers.
- */
- public static void finalizeDecodeInputs(final ByteBuffer[] decodeInputs,
- AlignedStripe alignedStripe) {
- for (int i = 0; i < alignedStripe.chunks.length; i++) {
- final StripingChunk chunk = alignedStripe.chunks[i];
- if (chunk != null && chunk.state == StripingChunk.FETCHED) {
- if (chunk.useChunkBuffer()) {
- chunk.getChunkBuffer().copyTo(decodeInputs[i]);
- } else {
- chunk.getByteBuffer().flip();
- }
- } else if (chunk != null && chunk.state == StripingChunk.ALLZERO) {
- //ZERO it. Will be better handled in other following issue.
- byte[] emptyBytes = new byte[decodeInputs[i].limit()];
- decodeInputs[i].put(emptyBytes);
- decodeInputs[i].flip();
- } else {
- decodeInputs[i] = null;
- }
- }
- }
-
- /**
- * Decode based on the given input buffers and erasure coding policy.
- */
- public static void decodeAndFillBuffer(final ByteBuffer[] decodeInputs,
- AlignedStripe alignedStripe, int dataBlkNum, int parityBlkNum,
- RawErasureDecoder decoder) {
- // Step 1: prepare indices and output buffers for missing data units
- int[] decodeIndices = new int[parityBlkNum];
- int pos = 0;
- for (int i = 0; i < dataBlkNum; i++) {
- if (alignedStripe.chunks[i] != null &&
- alignedStripe.chunks[i].state == StripingChunk.MISSING){
- decodeIndices[pos++] = i;
- }
- }
- decodeIndices = Arrays.copyOf(decodeIndices, pos);
- ByteBuffer[] decodeOutputs = new ByteBuffer[decodeIndices.length];
- for (int i = 0; i < decodeOutputs.length; i++) {
- decodeOutputs[i] = ByteBuffer.allocate(
- (int) alignedStripe.getSpanInBlock());
- }
-
- // Step 2: decode into prepared output buffers
- decoder.decode(decodeInputs, decodeIndices, decodeOutputs);
-
- // Step 3: fill original application buffer with decoded data
- for (int i = 0; i < decodeIndices.length; i++) {
- int missingBlkIdx = decodeIndices[i];
- StripingChunk chunk = alignedStripe.chunks[missingBlkIdx];
- if (chunk.state == StripingChunk.MISSING && chunk.useChunkBuffer()) {
- chunk.getChunkBuffer().copyFrom(decodeOutputs[i]);
- }
- }
- }
-
- /**
* Similar functionality with {@link #divideByteRangeIntoStripes}, but is used
* by stateful read and uses ByteBuffer as reading target buffer. Besides the
* read range is within a single stripe thus the calculation logic is simpler.
@@ -485,7 +387,7 @@ public class StripedBlockUtil {
/**
* Map the logical byte range to a set of inclusive {@link StripingCell}
* instances, each representing the overlap of the byte range to a cell
- * used by {@link DFSStripedOutputStream} in encoding
+ * used by {@link DFSStripedOutputStream} in encoding.
*/
@VisibleForTesting
private static StripingCell[] getStripingCellsOfByteRange(
@@ -530,7 +432,7 @@ public class StripedBlockUtil {
int dataBlkNum = ecPolicy.getNumDataUnits();
int parityBlkNum = ecPolicy.getNumParityUnits();
- VerticalRange ranges[] = new VerticalRange[dataBlkNum + parityBlkNum];
+ VerticalRange[] ranges = new VerticalRange[dataBlkNum + parityBlkNum];
long earliestStart = Long.MAX_VALUE;
long latestEnd = -1;
@@ -675,7 +577,7 @@ public class StripedBlockUtil {
@VisibleForTesting
static class StripingCell {
final ErasureCodingPolicy ecPolicy;
- /** Logical order in a block group, used when doing I/O to a block group */
+ /** Logical order in a block group, used when doing I/O to a block group. */
final int idxInBlkGroup;
final int idxInInternalBlk;
final int idxInStripe;
@@ -738,7 +640,7 @@ public class StripedBlockUtil {
*/
public static class AlignedStripe {
public VerticalRange range;
- /** status of each chunk in the stripe */
+ /** status of each chunk in the stripe. */
public final StripingChunk[] chunks;
public int fetchedChunksNum = 0;
public int missingChunksNum = 0;
@@ -790,9 +692,9 @@ public class StripedBlockUtil {
* +-----+
*/
public static class VerticalRange {
- /** start offset in the block group (inclusive) */
+ /** start offset in the block group (inclusive). */
public long offsetInBlock;
- /** length of the stripe range */
+ /** length of the stripe range. */
public long spanInBlock;
public VerticalRange(long offsetInBlock, long length) {
@@ -801,7 +703,7 @@ public class StripedBlockUtil {
this.spanInBlock = length;
}
- /** whether a position is in the range */
+ /** whether a position is in the range. */
public boolean include(long pos) {
return pos >= offsetInBlock && pos < offsetInBlock + spanInBlock;
}
@@ -915,7 +817,7 @@ public class StripedBlockUtil {
/**
* Note: target will be ready-to-read state after the call.
*/
- void copyTo(ByteBuffer target) {
+ public void copyTo(ByteBuffer target) {
for (ByteBuffer slice : slices) {
slice.flip();
target.put(slice);
@@ -923,7 +825,7 @@ public class StripedBlockUtil {
target.flip();
}
- void copyFrom(ByteBuffer src) {
+ public void copyFrom(ByteBuffer src) {
ByteBuffer tmp;
int len;
for (ByteBuffer slice : slices) {
@@ -970,6 +872,28 @@ public class StripedBlockUtil {
}
}
+ /** Used to indicate the buffered data's range in the block group. */
+ public static class StripeRange {
+ /** start offset in the block group (inclusive). */
+ final long offsetInBlock;
+ /** length of the stripe range. */
+ final long length;
+
+ public StripeRange(long offsetInBlock, long length) {
+ Preconditions.checkArgument(offsetInBlock >= 0 && length >= 0);
+ this.offsetInBlock = offsetInBlock;
+ this.length = length;
+ }
+
+ public boolean include(long pos) {
+ return pos >= offsetInBlock && pos < offsetInBlock + length;
+ }
+
+ public long getLength() {
+ return length;
+ }
+ }
+
/**
* Check if the information such as IDs and generation stamps in block-i
* match the block group.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/734d54c1/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
index 7d9d7dc..999eb1f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/util/TestStripedBlockUtil.java
@@ -283,5 +283,4 @@ public class TestStripedBlockUtil {
}
}
}
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: MAPREDUCE-6740. Enforce
mapreduce.task.timeout to be at least
mapreduce.task.progress-report.interval. (Haibo Chen via kasha)
Posted by ji...@apache.org.
MAPREDUCE-6740. Enforce mapreduce.task.timeout to be at least mapreduce.task.progress-report.interval. (Haibo Chen via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/537095d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/537095d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/537095d1
Branch: refs/heads/MAPREDUCE-6608
Commit: 537095d13cd38212ed162e0a360bdd9a8bd83498
Parents: 964e546
Author: Karthik Kambatla <ka...@apache.org>
Authored: Wed Sep 21 18:30:11 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Wed Sep 21 18:30:11 2016 -0700
----------------------------------------------------------------------
.../mapreduce/v2/app/TaskHeartbeatHandler.java | 24 ++++++-
.../v2/app/TestTaskHeartbeatHandler.java | 67 ++++++++++++++++++++
.../java/org/apache/hadoop/mapred/Task.java | 8 ++-
.../apache/hadoop/mapreduce/MRJobConfig.java | 9 ++-
.../hadoop/mapreduce/util/MRJobConfUtil.java | 16 +++++
5 files changed, 113 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/537095d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
index 303b4c1..6a716c7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/TaskHeartbeatHandler.java
@@ -23,10 +23,12 @@ import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.util.MRJobConfUtil;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
@@ -67,7 +69,7 @@ public class TaskHeartbeatHandler extends AbstractService {
//received from a task.
private Thread lostTaskCheckerThread;
private volatile boolean stopped;
- private int taskTimeOut = 5 * 60 * 1000;// 5 mins
+ private long taskTimeOut;
private int taskTimeOutCheckInterval = 30 * 1000; // 30 seconds.
private final EventHandler eventHandler;
@@ -87,7 +89,19 @@ public class TaskHeartbeatHandler extends AbstractService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- taskTimeOut = conf.getInt(MRJobConfig.TASK_TIMEOUT, 5 * 60 * 1000);
+ taskTimeOut = conf.getLong(
+ MRJobConfig.TASK_TIMEOUT, MRJobConfig.DEFAULT_TASK_TIMEOUT_MILLIS);
+
+ // enforce task timeout is at least twice as long as task report interval
+ long taskProgressReportIntervalMillis = MRJobConfUtil.
+ getTaskProgressReportInterval(conf);
+ long minimumTaskTimeoutAllowed = taskProgressReportIntervalMillis * 2;
+ if(taskTimeOut < minimumTaskTimeoutAllowed) {
+ taskTimeOut = minimumTaskTimeoutAllowed;
+ LOG.info("Task timeout must be as least twice as long as the task " +
+ "status report interval. Setting task timeout to " + taskTimeOut);
+ }
+
taskTimeOutCheckInterval =
conf.getInt(MRJobConfig.TASK_TIMEOUT_CHECK_INTERVAL_MS, 30 * 1000);
}
@@ -140,7 +154,7 @@ public class TaskHeartbeatHandler extends AbstractService {
while (iterator.hasNext()) {
Map.Entry<TaskAttemptId, ReportTime> entry = iterator.next();
- boolean taskTimedOut = (taskTimeOut > 0) &&
+ boolean taskTimedOut = (taskTimeOut > 0) &&
(currentTime > (entry.getValue().getLastProgress() + taskTimeOut));
if(taskTimedOut) {
@@ -163,4 +177,8 @@ public class TaskHeartbeatHandler extends AbstractService {
}
}
+ @VisibleForTesting
+ public long getTaskTimeOut() {
+ return taskTimeOut;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/537095d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestTaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestTaskHeartbeatHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestTaskHeartbeatHandler.java
index e9a0989..2623849 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestTaskHeartbeatHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestTaskHeartbeatHandler.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
+import org.junit.Assert;
import org.junit.Test;
@@ -50,6 +51,9 @@ public class TestTaskHeartbeatHandler {
Configuration conf = new Configuration();
conf.setInt(MRJobConfig.TASK_TIMEOUT, 10); //10 ms
+ // set TASK_PROGRESS_REPORT_INTERVAL to a value smaller than TASK_TIMEOUT
+ // so that TASK_TIMEOUT is not overridden
+ conf.setLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL, 5);
conf.setInt(MRJobConfig.TASK_TIMEOUT_CHECK_INTERVAL_MS, 10); //10 ms
hb.init(conf);
@@ -68,4 +72,67 @@ public class TestTaskHeartbeatHandler {
}
}
+ /**
+ * Test if the final heartbeat timeout is set correctly when task progress
+ * report interval is set bigger than the task timeout in the configuration.
+ */
+ @Test
+ public void testTaskTimeoutConfigSmallerThanTaskProgressReportInterval() {
+ testTaskTimeoutWrtProgressReportInterval(1000L, 5000L);
+ }
+
+ /**
+ * Test if the final heartbeat timeout is set correctly when task progress
+ * report interval is set smaller than the task timeout in the configuration.
+ */
+ @Test
+ public void testTaskTimeoutConfigBiggerThanTaskProgressReportInterval() {
+ testTaskTimeoutWrtProgressReportInterval(5000L, 1000L);
+ }
+
+ /**
+ * Test if the final heartbeat timeout is set correctly when task progress
+ * report interval is not set in the configuration.
+ */
+ @Test
+ public void testTaskTimeoutConfigWithoutTaskProgressReportInterval() {
+ final long taskTimeoutConfiged = 2000L;
+
+ final Configuration conf = new Configuration();
+ conf.setLong(MRJobConfig.TASK_TIMEOUT, taskTimeoutConfiged);
+
+ final long expectedTimeout = taskTimeoutConfiged;
+ verifyTaskTimeoutConfig(conf, expectedTimeout);
+ }
+
+ /**
+ * Test if task timeout is set properly in response to the configuration of
+ * the task progress report interval.
+ */
+ private static void testTaskTimeoutWrtProgressReportInterval(
+ long timeoutConfig, long taskreportInterval) {
+ final Configuration conf = new Configuration();
+ conf.setLong(MRJobConfig.TASK_TIMEOUT, timeoutConfig);
+ conf.setLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL, taskreportInterval);
+
+ // expected task timeout is at least twice as long as task report interval
+ final long expectedTimeout = Math.max(timeoutConfig, taskreportInterval*2);
+ verifyTaskTimeoutConfig(conf, expectedTimeout);
+ }
+
+ /**
+ * Verify task timeout is set as expected in TaskHeartBeatHandler with given
+ * configuration.
+ * @param conf the configuration
+ * @param expectedTimeout expected timeout value
+ */
+ private static void verifyTaskTimeoutConfig(final Configuration conf,
+ final long expectedTimeout) {
+ final TaskHeartbeatHandler hb =
+ new TaskHeartbeatHandler(null, SystemClock.getInstance(), 1);
+ hb.init(conf);
+
+ Assert.assertTrue("The value of the task timeout is incorrect.",
+ hb.getTaskTimeOut() == expectedTimeout);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/537095d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index d20e9bd..119d6a7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.lib.reduce.WrappedReducer;
import org.apache.hadoop.mapreduce.task.ReduceContextImpl;
+import org.apache.hadoop.mapreduce.util.MRJobConfUtil;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ExitUtil;
@@ -781,9 +782,10 @@ abstract public class Task implements Writable, Configurable {
int remainingRetries = MAX_RETRIES;
// get current flag value and reset it as well
boolean sendProgress = resetProgressFlag();
- long taskProgressInterval =
- conf.getLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL,
- MRJobConfig.DEFAULT_TASK_PROGRESS_REPORT_INTERVAL);
+
+ long taskProgressInterval = MRJobConfUtil.
+ getTaskProgressReportInterval(conf);
+
while (!taskDone.get()) {
synchronized (lock) {
done = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/537095d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index d740747..5716404 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -57,11 +57,6 @@ public interface MRJobConfig {
// negative values disable the limit
public static final long DEFAULT_TASK_LOCAL_WRITE_LIMIT_BYTES = -1;
- public static final String TASK_PROGRESS_REPORT_INTERVAL =
- "mapreduce.task.progress-report.interval";
- /** The number of milliseconds between progress reports. */
- public static final int DEFAULT_TASK_PROGRESS_REPORT_INTERVAL = 3000;
-
public static final String JAR = "mapreduce.job.jar";
public static final String ID = "mapreduce.job.id";
@@ -258,6 +253,10 @@ public interface MRJobConfig {
public static final String TASK_REDUCE_PROFILE_PARAMS = "mapreduce.task.profile.reduce.params";
public static final String TASK_TIMEOUT = "mapreduce.task.timeout";
+ long DEFAULT_TASK_TIMEOUT_MILLIS = 5 * 60 * 1000L;
+
+ String TASK_PROGRESS_REPORT_INTERVAL =
+ "mapreduce.task.progress-report.interval";
public static final String TASK_TIMEOUT_CHECK_INTERVAL_MS = "mapreduce.task.timeout.check-interval-ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/537095d1/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/MRJobConfUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/MRJobConfUtil.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/MRJobConfUtil.java
index 11d49a4..afedef3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/MRJobConfUtil.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/MRJobConfUtil.java
@@ -42,4 +42,20 @@ public final class MRJobConfUtil {
*/
private MRJobConfUtil() {
}
+
+ /**
+ * Get the progress heartbeat interval configuration for mapreduce tasks.
+ * By default, the value of progress heartbeat interval is a proportion of
+ * that of task timeout.
+ * @param conf the job configuration to read from
+ * @return the value of task progress report interval
+ */
+ public static long getTaskProgressReportInterval(final Configuration conf) {
+ long taskHeartbeatTimeOut = conf.getLong(
+ MRJobConfig.TASK_TIMEOUT, MRJobConfig.DEFAULT_TASK_TIMEOUT_MILLIS);
+ return conf.getLong(MRJobConfig.TASK_PROGRESS_REPORT_INTERVAL,
+ (long) (TASK_REPORT_INTERVAL_TO_TIMEOUT_RATIO * taskHeartbeatTimeOut));
+ }
+
+ public static final float TASK_REPORT_INTERVAL_TO_TIMEOUT_RATIO = 0.01f;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HADOOP-13601. Fix a log message
typo in AbstractDelegationTokenSecretManager. Contributed by Mehran Hassani.
Posted by ji...@apache.org.
HADOOP-13601. Fix a log message typo in AbstractDelegationTokenSecretManager. Contributed by Mehran Hassani.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e80386d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e80386d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e80386d6
Branch: refs/heads/MAPREDUCE-6608
Commit: e80386d69d5fb6a08aa3366e42d2518747af569f
Parents: 9f03b40
Author: Mingliang Liu <li...@apache.org>
Authored: Tue Sep 20 13:19:44 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Tue Sep 20 13:20:01 2016 -0700
----------------------------------------------------------------------
.../token/delegation/AbstractDelegationTokenSecretManager.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e80386d6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
index 1d7f2f5..cc2efc9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenSecretManager.java
@@ -528,7 +528,7 @@ extends AbstractDelegationTokenIdentifier>
DataInputStream in = new DataInputStream(buf);
TokenIdent id = createIdentifier();
id.readFields(in);
- LOG.info("Token cancelation requested for identifier: "+id);
+ LOG.info("Token cancellation requested for identifier: " + id);
if (id.getUser() == null) {
throw new InvalidToken("Token with no owner");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-5664. Fix Yarn documentation
to link to correct versions. Contributed by Xiao Chen
Posted by ji...@apache.org.
YARN-5664. Fix Yarn documentation to link to correct versions. Contributed by Xiao Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d16d2d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d16d2d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d16d2d7
Branch: refs/heads/MAPREDUCE-6608
Commit: 4d16d2d72f085e4efb6db27d397802b21df18815
Parents: 6eb700e
Author: Naganarasimha <na...@apache.org>
Authored: Sat Sep 24 23:21:03 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sat Sep 24 23:21:03 2016 +0530
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/CapacityScheduler.md | 2 +-
.../hadoop-yarn-site/src/site/markdown/NodeLabel.md | 2 +-
.../hadoop-yarn-site/src/site/markdown/ReservationSystem.md | 2 +-
.../src/site/markdown/WritingYarnApplications.md | 6 +++---
.../hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md | 6 +++---
5 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d16d2d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 6aa7007..9c9b03e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -39,7 +39,7 @@ Overview
The `CapacityScheduler` is designed to run Hadoop applications as a shared, multi-tenant cluster in an operator-friendly manner while maximizing the throughput and the utilization of the cluster.
-Traditionally each organization has it own private set of compute resources that have sufficient capacity to meet the organization's SLA under peak or near peak conditions. This generally leads to poor average utilization and overhead of managing multiple independent clusters, one per each organization. Sharing clusters between organizations is a cost-effective manner of running large Hadoop installations since this allows them to reap benefits of economies of scale without creating private clusters. However, organizations are concerned about sharing a cluster because they are worried about others using the resources that are critical for their SLAs.
+Traditionally each organization has it own private set of compute resources that have sufficient capacity to meet the organization's SLA under peak or near-peak conditions. This generally leads to poor average utilization and overhead of managing multiple independent clusters, one per each organization. Sharing clusters between organizations is a cost-effective manner of running large Hadoop installations since this allows them to reap benefits of economies of scale without creating private clusters. However, organizations are concerned about sharing a cluster because they are worried about others using the resources that are critical for their SLAs.
The `CapacityScheduler` is designed to allow sharing a large cluster while giving each organization capacity guarantees. The central idea is that the available resources in the Hadoop cluster are shared among multiple organizations who collectively fund the cluster based on their computing needs. There is an added benefit that an organization can access any excess capacity not being used by others. This provides elasticity for the organizations in a cost-effective manner.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d16d2d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
index 1fecf07..af75bfe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/NodeLabel.md
@@ -192,5 +192,5 @@ Following label-related fields can be seen on web UI:
Useful links
------------
-* [YARN Capacity Scheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html), if you need more understanding about how to configure Capacity Scheduler
+* [YARN Capacity Scheduler](./CapacityScheduler.html), if you need more understanding about how to configure Capacity Scheduler
* Write YARN application using node labels, you can see following two links as examples: [YARN distributed shell](https://issues.apache.org/jira/browse/YARN-2502), [Hadoop MapReduce](https://issues.apache.org/jira/browse/MAPREDUCE-6304)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d16d2d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md
index 51d2b97..7d1309f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ReservationSystem.md
@@ -64,4 +64,4 @@ With reference to the figure above, a typical reservation proceeds as follows:
Configuring the Reservation System
----------------------------------
-Configuring the `ReservationSystem` is simple. Currently we have added support for *reservations* in both `CapacityScheduler` and `FairScheduler`. You can mark any **leaf queue** in the **capacity-scheduler.xml** or **fair-scheduler.xml** as available for "reservations" (see [CapacityScheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Configuring_ReservationSystem_with_CapacityScheduler) and the [FairScheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html) for details). Then the capacity/fair share within that queue can be used for making reservations. Jobs can still be submitted to the *reservable queue* without a reservation, in which case they will be run in best-effort mode in whatever capacity is left over by the jobs running within active reservations.
+Configuring the `ReservationSystem` is simple. Currently we have added support for *reservations* in both `CapacityScheduler` and `FairScheduler`. You can mark any **leaf queue** in the **capacity-scheduler.xml** or **fair-scheduler.xml** as available for "reservations" (see [CapacityScheduler](./CapacityScheduler.html#Configuring_ReservationSystem_with_CapacityScheduler) and the [FairScheduler](./FairScheduler.html) for details). Then the capacity/fair share within that queue can be used for making reservations. Jobs can still be submitted to the *reservable queue* without a reservation, in which case they will be run in best-effort mode in whatever capacity is left over by the jobs running within active reservations.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d16d2d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
index 1aaec18..cdad3f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/WritingYarnApplications.md
@@ -577,11 +577,11 @@ Setting `-Djava.library.path` on the command line while launching a container ca
Useful Links
------------
-* [YARN Architecture](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html)
+* [YARN Architecture](./YARN.html)
-* [YARN Capacity Scheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html)
+* [YARN Capacity Scheduler](./CapacityScheduler.html)
-* [YARN Fair Scheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html)
+* [YARN Fair Scheduler](./FairScheduler.html)
Sample Code
-----------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d16d2d7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
index 974f41d..433c737 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/YARN.md
@@ -25,12 +25,12 @@ The per-application ApplicationMaster is, in effect, a framework specific librar
The ResourceManager has two main components: Scheduler and ApplicationsManager.
-The Scheduler is responsible for allocating resources to the various running applications subject to familiar constraints of capacities, queues etc. The Scheduler is pure scheduler in the sense that it performs no monitoring or tracking of status for the application. Also, it offers no guarantees about restarting failed tasks either due to application failure or hardware failures. The Scheduler performs its scheduling function based the resource requirements of the applications; it does so based on the abstract notion of a resource *Container* which incorporates elements such as memory, cpu, disk, network etc.
+The Scheduler is responsible for allocating resources to the various running applications subject to familiar constraints of capacities, queues etc. The Scheduler is pure scheduler in the sense that it performs no monitoring or tracking of status for the application. Also, it offers no guarantees about restarting failed tasks either due to application failure or hardware failures. The Scheduler performs its scheduling function based on the resource requirements of the applications; it does so based on the abstract notion of a resource *Container* which incorporates elements such as memory, cpu, disk, network etc.
-The Scheduler has a pluggable policy which is responsible for partitioning the cluster resources among the various queues, applications etc. The current schedulers such as the [CapacityScheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html) and the [FairScheduler](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/FairScheduler.html) would be some examples of plug-ins.
+The Scheduler has a pluggable policy which is responsible for partitioning the cluster resources among the various queues, applications etc. The current schedulers such as the [CapacityScheduler](./CapacityScheduler.html) and the [FairScheduler](./FairScheduler.html) would be some examples of plug-ins.
The ApplicationsManager is responsible for accepting job-submissions, negotiating the first container for executing the application specific ApplicationMaster and provides the service for restarting the ApplicationMaster container on failure. The per-application ApplicationMaster has the responsibility of negotiating appropriate resource containers from the Scheduler, tracking their status and monitoring for progress.
MapReduce in hadoop-2.x maintains **API compatibility** with previous stable release (hadoop-1.x). This means that all MapReduce jobs should still run unchanged on top of YARN with just a recompile.
-YARN also supports the notion of **resource reservation** via the [ReservationSystem](http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/ReservationSystem.html), a component that allows users to specify a profile of resources over-time and temporal constraints (e.g., deadlines), and reserve resources to ensure the predictable execution of important jobs.The *ReservationSystem* tracks resources over-time, performs admission control for reservations, and dynamically instruct the underlying scheduler to ensure that the reservation is fullfilled.
+YARN also supports the notion of **resource reservation** via the [ReservationSystem](./ReservationSystem.html), a component that allows users to specify a profile of resources over-time and temporal constraints (e.g., deadlines), and reserve resources to ensure the predictable execution of important jobs.The *ReservationSystem* tracks resources over-time, performs admission control for reservations, and dynamically instruct the underlying scheduler to ensure that the reservation is fullfilled.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDFS-9333. Some tests using
MiniDFSCluster errored complaining port in use. (iwasakims)
Posted by ji...@apache.org.
HDFS-9333. Some tests using MiniDFSCluster errored complaining port in use. (iwasakims)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/964e546a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/964e546a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/964e546a
Branch: refs/heads/MAPREDUCE-6608
Commit: 964e546ab1dba5f5d53b209ec6c9a70a85654765
Parents: 5a58bfe
Author: Masatake Iwasaki <iw...@apache.org>
Authored: Wed Sep 21 10:35:25 2016 +0900
Committer: Masatake Iwasaki <iw...@apache.org>
Committed: Wed Sep 21 10:35:25 2016 +0900
----------------------------------------------------------------------
.../blockmanagement/TestBlockTokenWithDFS.java | 8 ++++++-
.../TestBlockTokenWithDFSStriped.java | 23 +++++++++++++++++++-
.../hdfs/tools/TestDFSZKFailoverController.java | 18 ++++++++++-----
3 files changed, 42 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
index e7e7739..9374ae8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFS.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.log4j.Level;
@@ -349,7 +350,12 @@ public class TestBlockTokenWithDFS {
Configuration conf = getConf(numDataNodes);
try {
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes).build();
+ // prefer non-ephemeral port to avoid port collision on restartNameNode
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+ .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+ .numDataNodes(numDataNodes)
+ .build();
cluster.waitActive();
assertEquals(numDataNodes, cluster.getDataNodes().size());
doTestRead(conf, cluster, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
index 64a48c2..1714561 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockTokenWithDFSStriped.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedStripedBlock;
import org.apache.hadoop.hdfs.server.balancer.TestBalancer;
import org.apache.hadoop.hdfs.util.StripedBlockUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.Timeout;
@@ -59,7 +60,27 @@ public class TestBlockTokenWithDFSStriped extends TestBlockTokenWithDFS {
@Override
public void testRead() throws Exception {
conf = getConf();
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDNs).build();
+
+ /*
+ * prefer non-ephemeral port to avoid conflict with tests using
+ * ephemeral ports on MiniDFSCluster#restartDataNode(true).
+ */
+ Configuration[] overlays = new Configuration[numDNs];
+ for (int i = 0; i < overlays.length; i++) {
+ int offset = i * 10;
+ Configuration c = new Configuration();
+ c.set(DFSConfigKeys.DFS_DATANODE_ADDRESS_KEY, "127.0.0.1:"
+ + ServerSocketUtil.getPort(19866 + offset, 100));
+ c.set(DFSConfigKeys.DFS_DATANODE_IPC_ADDRESS_KEY, "127.0.0.1:"
+ + ServerSocketUtil.getPort(19867 + offset, 100));
+ overlays[i] = c;
+ }
+
+ cluster = new MiniDFSCluster.Builder(conf)
+ .nameNodePort(ServerSocketUtil.getPort(19820, 100))
+ .nameNodeHttpPort(ServerSocketUtil.getPort(19870, 100))
+ .numDataNodes(numDNs)
+ .build();
cluster.getFileSystem().getClient()
.setErasureCodingPolicy("/", null);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/964e546a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
index dfdcf34..bbb787e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDFSZKFailoverController.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
import org.apache.hadoop.hdfs.server.namenode.MockNameNodeResourceChecker;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.apache.hadoop.test.MultithreadedTestUtil.TestingThread;
@@ -75,14 +76,21 @@ public class TestDFSZKFailoverController extends ClientBaseWithFixes {
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
0);
-
- conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1", 10023);
- conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2", 10024);
+ // Get random port numbers in advance. Because ZKFCs and DFSHAAdmin
+ // needs rpc port numbers of all ZKFCs, Setting 0 does not work here.
+ conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn1",
+ ServerSocketUtil.getPort(10023, 100));
+ conf.setInt(DFSConfigKeys.DFS_HA_ZKFC_PORT_KEY + ".ns1.nn2",
+ ServerSocketUtil.getPort(10024, 100));
+
+ // prefer non-ephemeral port to avoid port collision on restartNameNode
MiniDFSNNTopology topology = new MiniDFSNNTopology()
.addNameservice(new MiniDFSNNTopology.NSConf("ns1")
- .addNN(new MiniDFSNNTopology.NNConf("nn1").setIpcPort(10021))
- .addNN(new MiniDFSNNTopology.NNConf("nn2").setIpcPort(10022)));
+ .addNN(new MiniDFSNNTopology.NNConf("nn1")
+ .setIpcPort(ServerSocketUtil.getPort(10021, 100)))
+ .addNN(new MiniDFSNNTopology.NNConf("nn2")
+ .setIpcPort(ServerSocketUtil.getPort(10022, 100))));
cluster = new MiniDFSCluster.Builder(conf)
.nnTopology(topology)
.numDataNodes(0)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: MAPREDUCE-6774. Add support for
HDFS erasure code policy to TestDFSIO. Contributed by Sammi Chen
Posted by ji...@apache.org.
MAPREDUCE-6774. Add support for HDFS erasure code policy to TestDFSIO. Contributed by Sammi Chen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/501a7785
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/501a7785
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/501a7785
Branch: refs/heads/MAPREDUCE-6608
Commit: 501a77856d6b6edfb261547117e719da7a9cd221
Parents: 58bae35
Author: Kai Zheng <ka...@intel.com>
Authored: Sun Sep 18 09:03:15 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Sun Sep 18 09:03:15 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/TestDFSIO.java | 159 +++++++++++++++----
1 file changed, 124 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/501a7785/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
index 05d4d77..d218169 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/fs/TestDFSIO.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.SequenceFile.CompressionType;
@@ -106,9 +107,14 @@ public class TestDFSIO implements Tool {
" [-nrFiles N]" +
" [-size Size[B|KB|MB|GB|TB]]" +
" [-resFile resultFileName] [-bufferSize Bytes]" +
- " [-storagePolicy storagePolicyName]";
+ " [-storagePolicy storagePolicyName]" +
+ " [-erasureCodePolicy erasureCodePolicyName]";
private Configuration config;
+ private static final String STORAGE_POLICY_NAME_KEY =
+ "test.io.block.storage.policy";
+ private static final String ERASURE_CODE_POLICY_NAME_KEY =
+ "test.io.erasure.code.policy";
static{
Configuration.addDefaultResource("hdfs-default.xml");
@@ -211,9 +217,9 @@ public class TestDFSIO implements Tool {
bench = new TestDFSIO();
bench.getConf().setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
cluster = new MiniDFSCluster.Builder(bench.getConf())
- .numDataNodes(2)
- .format(true)
- .build();
+ .numDataNodes(2)
+ .format(true)
+ .build();
FileSystem fs = cluster.getFileSystem();
bench.createControlFile(fs, DEFAULT_NR_BYTES, DEFAULT_NR_FILES);
@@ -356,7 +362,7 @@ public class TestDFSIO implements Tool {
ReflectionUtils.newInstance(codec, getConf());
}
- blockStoragePolicy = getConf().get("test.io.block.storage.policy", null);
+ blockStoragePolicy = getConf().get(STORAGE_POLICY_NAME_KEY, null);
}
@Override // IOMapperBase
@@ -388,9 +394,10 @@ public class TestDFSIO implements Tool {
*/
public static class WriteMapper extends IOStatMapper {
- public WriteMapper() {
- for(int i=0; i < bufferSize; i++)
- buffer[i] = (byte)('0' + i % 50);
+ public WriteMapper() {
+ for (int i = 0; i < bufferSize; i++) {
+ buffer[i] = (byte) ('0' + i % 50);
+ }
}
@Override // IOMapperBase
@@ -431,6 +438,9 @@ public class TestDFSIO implements Tool {
fs.delete(getDataDir(config), true);
fs.delete(writeDir, true);
long tStart = System.currentTimeMillis();
+ if (isECEnabled()) {
+ createAndEnableECOnPath(fs, getDataDir(config));
+ }
runIOTest(WriteMapper.class, writeDir);
long execTime = System.currentTimeMillis() - tStart;
return execTime;
@@ -734,6 +744,7 @@ public class TestDFSIO implements Tool {
TestType testType = null;
int bufferSize = DEFAULT_BUFFER_SIZE;
long nrBytes = 1*MEGA;
+ String erasureCodePolicyName = null;
int nrFiles = 1;
long skipSize = 0;
String resFileName = DEFAULT_RES_FILE_NAME;
@@ -785,26 +796,31 @@ public class TestDFSIO implements Tool {
resFileName = args[++i];
} else if (args[i].equalsIgnoreCase("-storagePolicy")) {
storagePolicy = args[++i];
+ } else if (args[i].equalsIgnoreCase("-erasureCodePolicy")) {
+ erasureCodePolicyName = args[++i];
} else {
System.err.println("Illegal argument: " + args[i]);
return -1;
}
}
- if(testType == null)
+ if (testType == null) {
return -1;
- if(testType == TestType.TEST_TYPE_READ_BACKWARD)
+ }
+ if (testType == TestType.TEST_TYPE_READ_BACKWARD) {
skipSize = -bufferSize;
- else if(testType == TestType.TEST_TYPE_READ_SKIP && skipSize == 0)
+ } else if (testType == TestType.TEST_TYPE_READ_SKIP && skipSize == 0) {
skipSize = bufferSize;
+ }
LOG.info("nrFiles = " + nrFiles);
LOG.info("nrBytes (MB) = " + toMB(nrBytes));
LOG.info("bufferSize = " + bufferSize);
- if(skipSize > 0)
+ if (skipSize > 0) {
LOG.info("skipSize = " + skipSize);
+ }
LOG.info("baseDir = " + getBaseDir(config));
- if(compressionClass != null) {
+ if (compressionClass != null) {
config.set("test.io.compression.class", compressionClass);
LOG.info("compressionClass = " + compressionClass);
}
@@ -813,31 +829,16 @@ public class TestDFSIO implements Tool {
config.setLong("test.io.skip.size", skipSize);
FileSystem fs = FileSystem.get(config);
- if (storagePolicy != null) {
- boolean isValid = false;
- Collection<BlockStoragePolicy> storagePolicies =
- ((DistributedFileSystem) fs).getAllStoragePolicies();
- try {
- for (BlockStoragePolicy policy : storagePolicies) {
- if (policy.getName().equals(storagePolicy)) {
- isValid = true;
- break;
- }
- }
- } catch (Exception e) {
- throw new IOException("Get block storage policies error: ", e);
- }
- if (!isValid) {
- System.out.println("Invalid block storage policy: " + storagePolicy);
- System.out.println("Current supported storage policy list: ");
- for (BlockStoragePolicy policy : storagePolicies) {
- System.out.println(policy.getName());
- }
+ if (erasureCodePolicyName != null) {
+ if (!checkErasureCodePolicy(erasureCodePolicyName, fs, testType)) {
return -1;
}
+ }
- config.set("test.io.block.storage.policy", storagePolicy);
- LOG.info("storagePolicy = " + storagePolicy);
+ if (storagePolicy != null) {
+ if (!checkStoragePolicy(storagePolicy, fs)) {
+ return -1;
+ }
}
if (isSequential) {
@@ -908,6 +909,94 @@ public class TestDFSIO implements Tool {
return ((float)bytes)/MEGA;
}
+ private boolean checkErasureCodePolicy(String erasureCodePolicyName,
+ FileSystem fs, TestType testType) throws IOException {
+ Collection<ErasureCodingPolicy> list =
+ ((DistributedFileSystem) fs).getAllErasureCodingPolicies();
+ boolean isValid = false;
+ for (ErasureCodingPolicy ec : list) {
+ if (erasureCodePolicyName.equals(ec.getName())) {
+ isValid = true;
+ break;
+ }
+ }
+
+ if (!isValid) {
+ System.out.println("Invalid erasure code policy: " +
+ erasureCodePolicyName);
+ System.out.println("Current supported erasure code policy list: ");
+ for (ErasureCodingPolicy ec : list) {
+ System.out.println(ec.getName());
+ }
+ return false;
+ }
+
+ if (testType == TestType.TEST_TYPE_APPEND ||
+ testType == TestType.TEST_TYPE_TRUNCATE) {
+ System.out.println("So far append or truncate operation" +
+ " does not support erasureCodePolicy");
+ return false;
+ }
+
+ config.set(ERASURE_CODE_POLICY_NAME_KEY, erasureCodePolicyName);
+ LOG.info("erasureCodePolicy = " + erasureCodePolicyName);
+ return true;
+ }
+
+ private boolean checkStoragePolicy(String storagePolicy, FileSystem fs)
+ throws IOException {
+ boolean isValid = false;
+ Collection<BlockStoragePolicy> storagePolicies =
+ ((DistributedFileSystem) fs).getAllStoragePolicies();
+ try {
+ for (BlockStoragePolicy policy : storagePolicies) {
+ if (policy.getName().equals(storagePolicy)) {
+ isValid = true;
+ break;
+ }
+ }
+ } catch (Exception e) {
+ throw new IOException("Get block storage policies error: ", e);
+ }
+
+ if (!isValid) {
+ System.out.println("Invalid block storage policy: " + storagePolicy);
+ System.out.println("Current supported storage policy list: ");
+ for (BlockStoragePolicy policy : storagePolicies) {
+ System.out.println(policy.getName());
+ }
+ return false;
+ }
+
+ config.set(STORAGE_POLICY_NAME_KEY, storagePolicy);
+ LOG.info("storagePolicy = " + storagePolicy);
+ return true;
+ }
+
+ private boolean isECEnabled() {
+ String erasureCodePolicyName =
+ getConf().get(ERASURE_CODE_POLICY_NAME_KEY, null);
+ return erasureCodePolicyName != null ? true : false;
+ }
+
+ void createAndEnableECOnPath(FileSystem fs, Path path)
+ throws IOException {
+ String erasureCodePolicyName =
+ getConf().get(ERASURE_CODE_POLICY_NAME_KEY, null);
+
+ fs.mkdirs(path);
+ Collection<ErasureCodingPolicy> list =
+ ((DistributedFileSystem) fs).getAllErasureCodingPolicies();
+ for (ErasureCodingPolicy ec : list) {
+ if (erasureCodePolicyName.equals(ec.getName())) {
+ ((DistributedFileSystem) fs).setErasureCodingPolicy(path, ec);
+ LOG.info("enable erasureCodePolicy = " + erasureCodePolicyName +
+ " on " + path.toString());
+ break;
+ }
+ }
+ }
+
private void analyzeResult( FileSystem fs,
TestType testType,
long execTime,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-5163. Migrate
TestClientToAMTokens and TestClientRMTokens tests from the old RPC engine.
Contributed by Wei Zhou and Kai Zheng
Posted by ji...@apache.org.
YARN-5163. Migrate TestClientToAMTokens and TestClientRMTokens tests from the old RPC engine. Contributed by Wei Zhou and Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/58bae354
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/58bae354
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/58bae354
Branch: refs/heads/MAPREDUCE-6608
Commit: 58bae3544747ad47373925add35b39bda8be8c5a
Parents: 8a40953
Author: Kai Zheng <ka...@intel.com>
Authored: Sun Sep 18 08:43:36 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Sun Sep 18 08:43:36 2016 +0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +
.../security/TestClientToAMTokens.java | 108 ++++++++++---------
3 files changed, 65 insertions(+), 51 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58bae354/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index 3c5885f..0d2f975 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -31,8 +31,6 @@ import org.junit.Assert;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.RetryPolicy;
-import org.apache.hadoop.ipc.protobuf.TestProtos;
-import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.token.Token;
@@ -481,7 +479,7 @@ public class TestRpcBase {
}
}
- protected static TestProtos.EmptyRequestProto newEmptyRequest() {
+ public static TestProtos.EmptyRequestProto newEmptyRequest() {
return TestProtos.EmptyRequestProto.newBuilder().build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58bae354/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 06f6c4f..32eafeb 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -67,3 +67,7 @@ service NewerProtobufRpcProto {
rpc ping(EmptyRequestProto) returns (EmptyResponseProto);
rpc echo(EmptyRequestProto) returns (EmptyResponseProto);
}
+
+service CustomProto {
+ rpc ping(EmptyRequestProto) returns (EmptyResponseProto);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/58bae354/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
index d36fb9f..a451356 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestClientToAMTokens.java
@@ -18,33 +18,19 @@
package org.apache.hadoop.yarn.server.resourcemanager.security;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.resourcemanager
- .ParameterizedSchedulerTestBase;
-import static org.junit.Assert.fail;
-import org.junit.Before;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.lang.annotation.Annotation;
-import java.net.InetSocketAddress;
-import java.nio.ByteBuffer;
-import java.security.PrivilegedAction;
-import java.security.PrivilegedExceptionAction;
-import java.util.Timer;
-import java.util.TimerTask;
-
-import javax.security.sasl.SaslException;
-
-import org.junit.Assert;
-
+import com.google.protobuf.BlockingService;
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolInfo;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.TestRpcBase;
+import org.apache.hadoop.ipc.protobuf.TestProtos;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.SecurityInfo;
@@ -63,9 +49,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.ClientToAMTokenSecretManager;
@@ -75,12 +61,30 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRMWithCustomAMLauncher;
+import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import javax.security.sasl.SaslException;
+import java.io.IOException;
+import java.lang.annotation.Annotation;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.security.PrivilegedAction;
+import java.security.PrivilegedExceptionAction;
+import java.util.Timer;
+import java.util.TimerTask;
+
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
private YarnConfiguration conf;
@@ -89,11 +93,12 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
conf = getConf();
}
- private interface CustomProtocol {
- @SuppressWarnings("unused")
- public static final long versionID = 1L;
-
- public void ping() throws YarnException, IOException;
+ @TokenInfo(ClientToAMTokenSelector.class)
+ @ProtocolInfo(protocolName =
+ "org.apache.hadoop.yarn.server.resourcemanager.security$CustomProtocol",
+ protocolVersion = 1)
+ public interface CustomProtocol
+ extends TestRpcServiceProtos.CustomProto.BlockingInterface {
}
private static class CustomSecurityInfo extends SecurityInfo {
@@ -137,8 +142,10 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
}
@Override
- public void ping() throws YarnException, IOException {
+ public TestProtos.EmptyResponseProto ping(RpcController unused,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
this.pinged = true;
+ return TestProtos.EmptyResponseProto.newBuilder().build();
}
public ClientToAMTokenSecretManager getClientToAMTokenSecretManager() {
@@ -148,6 +155,13 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
@Override
protected void serviceStart() throws Exception {
Configuration conf = getConfig();
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(conf, CustomProtocol.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+
+ BlockingService service = TestRpcServiceProtos.CustomProto
+ .newReflectiveBlockingService(this);
Server server;
try {
@@ -158,7 +172,7 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
.setProtocol(CustomProtocol.class)
.setNumHandlers(1)
.setSecretManager(secretMgr)
- .setInstance(this).build();
+ .setInstance(service).build();
} catch (Exception e) {
throw new YarnRuntimeException(e);
}
@@ -176,6 +190,8 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
public void testClientToAMTokens() throws Exception {
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION,
"kerberos");
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(conf, CustomProtocol.class, ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(conf);
ContainerManagementProtocol containerManager =
@@ -268,10 +284,9 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
// Verify denial for unauthenticated user
try {
- CustomProtocol client =
- (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
- conf);
- client.ping();
+ CustomProtocol client = RPC.getProxy(CustomProtocol.class,
+ 1L, am.address, conf);
+ client.ping(null, TestRpcBase.newEmptyRequest());
fail("Access by unauthenticated user should fail!!");
} catch (Exception e) {
Assert.assertFalse(am.pinged);
@@ -335,17 +350,16 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
@Override
public Void run() throws Exception {
try {
- CustomProtocol client =
- (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L,
+ CustomProtocol client = RPC.getProxy(CustomProtocol.class, 1L,
am.address, conf);
- client.ping();
+ client.ping(null, TestRpcBase.newEmptyRequest());
fail("Connection initiation with illegally modified "
+ "tokens is expected to fail.");
return null;
- } catch (YarnException ex) {
- fail("Cannot get a YARN remote exception as "
- + "it will indicate RPC success");
- throw ex;
+ } catch (ServiceException ex) {
+ //fail("Cannot get a YARN remote exception as "
+ // + "it indicates RPC success");
+ throw (Exception) ex.getCause();
}
}
});
@@ -383,9 +397,8 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
@Override
public Void run() throws Exception {
CustomProtocol client =
- (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
- conf);
- client.ping();
+ RPC.getProxy(CustomProtocol.class, 1L, am.address, conf);
+ client.ping(null, TestRpcBase.newEmptyRequest());
Assert.assertTrue(am.pinged);
return null;
}
@@ -402,10 +415,9 @@ public class TestClientToAMTokens extends ParameterizedSchedulerTestBase {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- CustomProtocol client =
- (CustomProtocol) RPC.getProxy(CustomProtocol.class, 1L, am.address,
- conf);
- client.ping();
+ CustomProtocol client = RPC.getProxy(CustomProtocol.class,
+ 1L, am.address, conf);
+ client.ping(null, TestRpcBase.newEmptyRequest());
Assert.assertTrue(am.pinged);
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HDFS-10894. Remove the redundant
charactors for command -saveNamespace in HDFSCommands.md Contributed by Yiqun
Lin.
Posted by ji...@apache.org.
HDFS-10894. Remove the redundant charactors for command -saveNamespace in HDFSCommands.md Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4480f7f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4480f7f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4480f7f
Branch: refs/heads/MAPREDUCE-6608
Commit: c4480f7f42f30f40900a73506790cd9bce3a823c
Parents: e5ef51e
Author: Brahma Reddy Battula <br...@apache.org>
Authored: Fri Sep 23 19:11:02 2016 +0530
Committer: Brahma Reddy Battula <br...@apache.org>
Committed: Fri Sep 23 19:11:02 2016 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4480f7f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 83035bb..ec94afd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -402,7 +402,7 @@ Usage:
|:---- |:---- |
| `-report` `[-live]` `[-dead]` `[-decommissioning]` | Reports basic filesystem information and statistics, The dfs usage can be different from "du" usage, because it measures raw space used by replication, checksums, snapshots and etc. on all the DNs. Optional flags may be used to filter the list of displayed DataNodes. |
| `-safemode` enter\|leave\|get\|wait\|forceExit | Safe mode maintenance command. Safe mode is a Namenode state in which it <br/>1. does not accept changes to the name space (read-only) <br/>2. does not replicate or delete blocks. <br/>Safe mode is entered automatically at Namenode startup, and leaves safe mode automatically when the configured minimum percentage of blocks satisfies the minimum replication condition. If Namenode detects any anomaly then it will linger in safe mode till that issue is resolved. If that anomaly is the consequence of a deliberate action, then administrator can use -safemode forceExit to exit safe mode. The cases where forceExit may be required are<br/> 1. Namenode metadata is not consistent. If Namenode detects that metadata has been modified out of band and can cause data loss, then Namenode will enter forceExit state. At that point user can either restart Namenode with correct metadata files or forceExit (if data loss is acceptable).<br/>2. Rollback c
auses metadata to be replaced and rarely it can trigger safe mode forceExit state in Namenode. In that case you may proceed by issuing -safemode forceExit.<br/> Safe mode can also be entered manually, but then it can only be turned off manually as well. |
-| `-saveNamespace` `\[-beforeShutdown\]` | Save current namespace into storage directories and reset edits log. Requires safe mode. If the "beforeShutdown" option is given, the NameNode does a checkpoint if and only if no checkpoint has been done during a time window (a configurable number of checkpoint periods). This is usually used before shutting down the NameNode to prevent potential fsimage/editlog corruption. |
+| `-saveNamespace` `[-beforeShutdown]` | Save current namespace into storage directories and reset edits log. Requires safe mode. If the "beforeShutdown" option is given, the NameNode does a checkpoint if and only if no checkpoint has been done during a time window (a configurable number of checkpoint periods). This is usually used before shutting down the NameNode to prevent potential fsimage/editlog corruption. |
| `-rollEdits` | Rolls the edit log on the active NameNode. |
| `-restoreFailedStorage` true\|false\|check | This option will turn on/off automatic attempt to restore failed storage replicas. If a failed storage becomes available again the system will attempt to restore edits and/or fsimage during checkpoint. 'check' option will return current setting. |
| `-refreshNodes` | Re-read the hosts and exclude files to update the set of Datanodes that are allowed to connect to the Namenode and those that should be decommissioned or recommissioned. |
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HADOOP-13169. Randomize file list
in SimpleCopyListing. Contributed by Rajesh Balamohan.
Posted by ji...@apache.org.
HADOOP-13169. Randomize file list in SimpleCopyListing. Contributed by Rajesh Balamohan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98bdb513
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98bdb513
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98bdb513
Branch: refs/heads/MAPREDUCE-6608
Commit: 98bdb5139769eb55893971b43b9c23da9513a784
Parents: 7558dbb
Author: Chris Nauroth <cn...@apache.org>
Authored: Mon Sep 19 15:16:47 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Mon Sep 19 15:16:47 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/tools/DistCpConstants.java | 4 +
.../apache/hadoop/tools/SimpleCopyListing.java | 114 +++++++++++++++++--
.../apache/hadoop/tools/TestCopyListing.java | 83 +++++++++++++-
3 files changed, 189 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98bdb513/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index 95d26df..96f364c 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -58,6 +58,10 @@ public class DistCpConstants {
public static final String CONF_LABEL_APPEND = "distcp.copy.append";
public static final String CONF_LABEL_DIFF = "distcp.copy.diff";
public static final String CONF_LABEL_BANDWIDTH_MB = "distcp.map.bandwidth.mb";
+ public static final String CONF_LABEL_SIMPLE_LISTING_FILESTATUS_SIZE =
+ "distcp.simplelisting.file.status.size";
+ public static final String CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES =
+ "distcp.simplelisting.randomize.files";
public static final String CONF_LABEL_FILTERS_FILE =
"distcp.filters.file";
public static final String CONF_LABEL_MAX_CHUNKS_TOLERABLE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98bdb513/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
index 3f52203..bc30aa1 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/SimpleCopyListing.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.tools;
+import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
@@ -42,7 +43,10 @@ import com.google.common.annotations.VisibleForTesting;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
+import java.util.List;
+import java.util.Random;
import static org.apache.hadoop.tools.DistCpConstants
.HDFS_RESERVED_RAW_DIRECTORY_NAME;
@@ -56,13 +60,19 @@ import static org.apache.hadoop.tools.DistCpConstants
public class SimpleCopyListing extends CopyListing {
private static final Log LOG = LogFactory.getLog(SimpleCopyListing.class);
+ public static final int DEFAULT_FILE_STATUS_SIZE = 1000;
+ public static final boolean DEFAULT_RANDOMIZE_FILE_LISTING = true;
+
private long totalPaths = 0;
private long totalDirs = 0;
private long totalBytesToCopy = 0;
private int numListstatusThreads = 1;
+ private final int fileStatusLimit;
+ private final boolean randomizeFileListing;
private final int maxRetries = 3;
private CopyFilter copyFilter;
private DistCpSync distCpSync;
+ private final Random rnd = new Random();
/**
* Protected constructor, to initialize configuration.
@@ -76,6 +86,17 @@ public class SimpleCopyListing extends CopyListing {
numListstatusThreads = getConf().getInt(
DistCpConstants.CONF_LABEL_LISTSTATUS_THREADS,
DistCpConstants.DEFAULT_LISTSTATUS_THREADS);
+ fileStatusLimit = Math.max(1, getConf()
+ .getInt(DistCpConstants.CONF_LABEL_SIMPLE_LISTING_FILESTATUS_SIZE,
+ DEFAULT_FILE_STATUS_SIZE));
+ randomizeFileListing = getConf().getBoolean(
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES,
+ DEFAULT_RANDOMIZE_FILE_LISTING);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("numListstatusThreads=" + numListstatusThreads
+ + ", fileStatusLimit=" + fileStatusLimit
+ + ", randomizeFileListing=" + randomizeFileListing);
+ }
copyFilter = CopyFilter.getCopyFilter(getConf());
copyFilter.initialize();
}
@@ -83,9 +104,13 @@ public class SimpleCopyListing extends CopyListing {
@VisibleForTesting
protected SimpleCopyListing(Configuration configuration,
Credentials credentials,
- int numListstatusThreads) {
+ int numListstatusThreads,
+ int fileStatusLimit,
+ boolean randomizeFileListing) {
super(configuration, credentials);
this.numListstatusThreads = numListstatusThreads;
+ this.fileStatusLimit = Math.max(1, fileStatusLimit);
+ this.randomizeFileListing = randomizeFileListing;
}
protected SimpleCopyListing(Configuration configuration,
@@ -236,6 +261,7 @@ public class SimpleCopyListing extends CopyListing {
FileSystem sourceFS = sourceRoot.getFileSystem(getConf());
try {
+ List<FileStatusInfo> fileStatuses = Lists.newArrayList();
for (DiffInfo diff : diffList) {
// add snapshot paths prefix
diff.target = new Path(options.getSourcePaths().get(0), diff.target);
@@ -259,10 +285,13 @@ public class SimpleCopyListing extends CopyListing {
sourceDirs.add(sourceStatus);
traverseDirectory(fileListWriter, sourceFS, sourceDirs,
- sourceRoot, options, excludeList);
+ sourceRoot, options, excludeList, fileStatuses);
}
}
}
+ if (randomizeFileListing) {
+ writeToFileListing(fileStatuses, fileListWriter);
+ }
fileListWriter.close();
fileListWriter = null;
} finally {
@@ -296,6 +325,7 @@ public class SimpleCopyListing extends CopyListing {
}
try {
+ List<FileStatusInfo> statusList = Lists.newArrayList();
for (Path path: options.getSourcePaths()) {
FileSystem sourceFS = path.getFileSystem(getConf());
final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
@@ -326,8 +356,14 @@ public class SimpleCopyListing extends CopyListing {
preserveAcls && sourceStatus.isDirectory(),
preserveXAttrs && sourceStatus.isDirectory(),
preserveRawXAttrs && sourceStatus.isDirectory());
- writeToFileListing(fileListWriter, sourceCopyListingStatus,
- sourcePathRoot);
+ if (randomizeFileListing) {
+ addToFileListing(statusList,
+ new FileStatusInfo(sourceCopyListingStatus, sourcePathRoot),
+ fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, sourceCopyListingStatus,
+ sourcePathRoot);
+ }
if (sourceStatus.isDirectory()) {
if (LOG.isDebugEnabled()) {
@@ -337,9 +373,12 @@ public class SimpleCopyListing extends CopyListing {
}
}
traverseDirectory(fileListWriter, sourceFS, sourceDirs,
- sourcePathRoot, options, null);
+ sourcePathRoot, options, null, statusList);
}
}
+ if (randomizeFileListing) {
+ writeToFileListing(statusList, fileListWriter);
+ }
fileListWriter.close();
printStats();
LOG.info("Build file listing completed.");
@@ -349,6 +388,52 @@ public class SimpleCopyListing extends CopyListing {
}
}
+ private void addToFileListing(List<FileStatusInfo> fileStatusInfoList,
+ FileStatusInfo statusInfo, SequenceFile.Writer fileListWriter)
+ throws IOException {
+ fileStatusInfoList.add(statusInfo);
+ if (fileStatusInfoList.size() > fileStatusLimit) {
+ writeToFileListing(fileStatusInfoList, fileListWriter);
+ }
+ }
+
+ @VisibleForTesting
+ void setSeedForRandomListing(long seed) {
+ this.rnd.setSeed(seed);
+ }
+
+ private void writeToFileListing(List<FileStatusInfo> fileStatusInfoList,
+ SequenceFile.Writer fileListWriter) throws IOException {
+ /**
+ * In cloud storage systems, it is possible to get region hotspot.
+ * Shuffling paths can avoid such cases and also ensure that
+ * some mappers do not get lots of similar paths.
+ */
+ Collections.shuffle(fileStatusInfoList, rnd);
+ for (FileStatusInfo fileStatusInfo : fileStatusInfoList) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding " + fileStatusInfo.fileStatus.getPath());
+ }
+ writeToFileListing(fileListWriter, fileStatusInfo.fileStatus,
+ fileStatusInfo.sourceRootPath);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Number of paths written to fileListing="
+ + fileStatusInfoList.size());
+ }
+ fileStatusInfoList.clear();
+ }
+
+ private static class FileStatusInfo {
+ private CopyListingFileStatus fileStatus;
+ private Path sourceRootPath;
+
+ FileStatusInfo(CopyListingFileStatus fileStatus, Path sourceRootPath) {
+ this.fileStatus = fileStatus;
+ this.sourceRootPath = sourceRootPath;
+ }
+ }
+
private Path computeSourceRootPath(FileStatus sourceStatus,
DistCpOptions options) throws IOException {
@@ -516,15 +601,18 @@ public class SimpleCopyListing extends CopyListing {
ArrayList<FileStatus> sourceDirs,
Path sourcePathRoot,
DistCpOptions options,
- HashSet<String> excludeList)
+ HashSet<String> excludeList,
+ List<FileStatusInfo> fileStatuses)
throws IOException {
final boolean preserveAcls = options.shouldPreserve(FileAttribute.ACL);
final boolean preserveXAttrs = options.shouldPreserve(FileAttribute.XATTR);
final boolean preserveRawXattrs = options.shouldPreserveRawXattrs();
assert numListstatusThreads > 0;
- LOG.debug("Starting thread pool of " + numListstatusThreads +
- " listStatus workers.");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Starting thread pool of " + numListstatusThreads +
+ " listStatus workers.");
+ }
ProducerConsumer<FileStatus, FileStatus[]> workers =
new ProducerConsumer<FileStatus, FileStatus[]>(numListstatusThreads);
for (int i = 0; i < numListstatusThreads; i++) {
@@ -551,8 +639,14 @@ public class SimpleCopyListing extends CopyListing {
preserveAcls && child.isDirectory(),
preserveXAttrs && child.isDirectory(),
preserveRawXattrs && child.isDirectory());
- writeToFileListing(fileListWriter, childCopyListingStatus,
- sourcePathRoot);
+ if (randomizeFileListing) {
+ addToFileListing(fileStatuses,
+ new FileStatusInfo(childCopyListingStatus, sourcePathRoot),
+ fileListWriter);
+ } else {
+ writeToFileListing(fileListWriter, childCopyListingStatus,
+ sourcePathRoot);
+ }
}
if (retry < maxRetries) {
if (child.isDirectory()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98bdb513/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
index 896763d..ea63e23 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestCopyListing.java
@@ -25,7 +25,6 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter;
import org.apache.hadoop.tools.util.TestDistCpUtils;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.security.Credentials;
@@ -46,7 +45,9 @@ import java.io.OutputStream;
import java.util.Arrays;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
+import java.util.Random;
@RunWith(value = Parameterized.class)
public class TestCopyListing extends SimpleCopyListing {
@@ -77,7 +78,7 @@ public class TestCopyListing extends SimpleCopyListing {
}
public TestCopyListing(int numListstatusThreads) {
- super(config, CREDENTIALS, numListstatusThreads);
+ super(config, CREDENTIALS, numListstatusThreads, 0, false);
}
protected TestCopyListing(Configuration configuration) {
@@ -221,6 +222,84 @@ public class TestCopyListing extends SimpleCopyListing {
}
}
+ @Test(timeout=60000)
+ public void testWithRandomFileListing() throws IOException {
+ FileSystem fs = null;
+ try {
+ fs = FileSystem.get(getConf());
+ List<Path> srcPaths = new ArrayList<>();
+ List<Path> srcFiles = new ArrayList<>();
+ Path target = new Path("/tmp/out/1");
+ final int pathCount = 25;
+ for (int i = 0; i < pathCount; i++) {
+ Path p = new Path("/tmp", String.valueOf(i));
+ srcPaths.add(p);
+ fs.mkdirs(p);
+
+ Path fileName = new Path(p, i + ".txt");
+ srcFiles.add(fileName);
+ try (OutputStream out = fs.create(fileName)) {
+ out.write(i);
+ }
+ }
+
+ Path listingFile = new Path("/tmp/file");
+ DistCpOptions options = new DistCpOptions(srcPaths, target);
+ options.setSyncFolder(true);
+
+ // Check without randomizing files
+ getConf().setBoolean(
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, false);
+ SimpleCopyListing listing = new SimpleCopyListing(getConf(), CREDENTIALS);
+ listing.buildListing(listingFile, options);
+
+ Assert.assertEquals(listing.getNumberOfPaths(), pathCount);
+ validateFinalListing(listingFile, srcFiles);
+ fs.delete(listingFile, true);
+
+ // Check with randomized file listing
+ getConf().setBoolean(
+ DistCpConstants.CONF_LABEL_SIMPLE_LISTING_RANDOMIZE_FILES, true);
+ listing = new SimpleCopyListing(getConf(), CREDENTIALS);
+
+ // Set the seed for randomness, so that it can be verified later
+ long seed = System.nanoTime();
+ listing.setSeedForRandomListing(seed);
+ listing.buildListing(listingFile, options);
+ Assert.assertEquals(listing.getNumberOfPaths(), pathCount);
+
+ // validate randomness
+ Collections.shuffle(srcFiles, new Random(seed));
+ validateFinalListing(listingFile, srcFiles);
+ } finally {
+ TestDistCpUtils.delete(fs, "/tmp");
+ }
+ }
+
+ private void validateFinalListing(Path pathToListFile, List<Path> srcFiles)
+ throws IOException {
+ FileSystem fs = pathToListFile.getFileSystem(config);
+
+ try (SequenceFile.Reader reader = new SequenceFile.Reader(
+ config, SequenceFile.Reader.file(pathToListFile))) {
+ CopyListingFileStatus currentVal = new CopyListingFileStatus();
+
+ Text currentKey = new Text();
+ int idx = 0;
+ while (reader.next(currentKey)) {
+ reader.getCurrentValue(currentVal);
+ Assert.assertEquals("srcFiles.size=" + srcFiles.size()
+ + ", idx=" + idx, fs.makeQualified(srcFiles.get(idx)),
+ currentVal.getPath());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("val=" + fs.makeQualified(srcFiles.get(idx)));
+ }
+ idx++;
+ }
+ }
+ }
+
+
@Test(timeout=10000)
public void testBuildListingForSingleFile() {
FileSystem fs = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-3141. Improve locks in
SchedulerApplicationAttempt/FSAppAttempt/FiCaSchedulerApp. Contributed by
Wangda Tan
Posted by ji...@apache.org.
YARN-3141. Improve locks in SchedulerApplicationAttempt/FSAppAttempt/FiCaSchedulerApp. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8a30f2f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8a30f2f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8a30f2f
Branch: refs/heads/MAPREDUCE-6608
Commit: b8a30f2f170ffbd590e7366c3c944ab4919e40df
Parents: ea29e3b
Author: Jian He <ji...@apache.org>
Authored: Mon Sep 19 16:58:39 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Sep 19 17:08:01 2016 +0800
----------------------------------------------------------------------
.../scheduler/SchedulerApplicationAttempt.java | 744 +++++++++++--------
.../allocator/RegularContainerAllocator.java | 2 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 418 ++++++-----
.../scheduler/fair/FSAppAttempt.java | 465 ++++++------
4 files changed, 922 insertions(+), 707 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a30f2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 97d29cf..adc3a97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -26,8 +26,11 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import com.google.common.collect.ConcurrentHashMultiset;
import org.apache.commons.lang.time.DateUtils;
import org.apache.commons.lang.time.FastDateFormat;
import org.apache.commons.logging.Log;
@@ -71,8 +74,6 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import com.google.common.collect.HashMultiset;
-import com.google.common.collect.Multiset;
/**
* Represents an application attempt from the viewpoint of the scheduler.
@@ -97,14 +98,14 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
protected final AppSchedulingInfo appSchedulingInfo;
protected ApplicationAttemptId attemptId;
protected Map<ContainerId, RMContainer> liveContainers =
- new HashMap<ContainerId, RMContainer>();
+ new ConcurrentHashMap<>();
protected final Map<SchedulerRequestKey, Map<NodeId, RMContainer>>
reservedContainers = new HashMap<>();
- private final Multiset<SchedulerRequestKey> reReservations =
- HashMultiset.create();
+ private final ConcurrentHashMultiset<SchedulerRequestKey> reReservations =
+ ConcurrentHashMultiset.create();
- private Resource resourceLimit = Resource.newInstance(0, 0);
+ private volatile Resource resourceLimit = Resource.newInstance(0, 0);
private boolean unmanagedAM = true;
private boolean amRunning = false;
private LogAggregationContext logAggregationContext;
@@ -138,8 +139,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
* the application successfully schedules a task (at rack or node local), it
* is reset to 0.
*/
- Multiset<SchedulerRequestKey> schedulingOpportunities = HashMultiset.create();
-
+ private ConcurrentHashMultiset<SchedulerRequestKey> schedulingOpportunities =
+ ConcurrentHashMultiset.create();
+
/**
* Count how many times the application has been given an opportunity to
* schedule a non-partitioned resource request at each priority. Each time the
@@ -147,15 +149,16 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
* incremented, and each time the application successfully schedules a task,
* it is reset to 0 when schedule any task at corresponding priority.
*/
- Multiset<SchedulerRequestKey> missedNonPartitionedReqSchedulingOpportunity =
- HashMultiset.create();
+ private ConcurrentHashMultiset<SchedulerRequestKey>
+ missedNonPartitionedReqSchedulingOpportunity =
+ ConcurrentHashMultiset.create();
// Time of the last container scheduled at the current allowed level
protected Map<SchedulerRequestKey, Long> lastScheduledContainer =
- new HashMap<>();
+ new ConcurrentHashMap<>();
- protected Queue queue;
- protected boolean isStopped = false;
+ protected volatile Queue queue;
+ protected volatile boolean isStopped = false;
protected String appAMNodePartitionName = CommonNodeLabelsManager.NO_LABEL;
@@ -163,6 +166,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private RMAppAttempt appAttempt;
+ protected ReentrantReadWriteLock.ReadLock readLock;
+ protected ReentrantReadWriteLock.WriteLock writeLock;
+
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
RMContext rmContext) {
@@ -188,14 +194,23 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
appSubmissionContext.getLogAggregationContext();
}
}
+
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
}
/**
* Get the live containers of the application.
* @return live containers of the application
*/
- public synchronized Collection<RMContainer> getLiveContainers() {
- return new ArrayList<RMContainer>(liveContainers.values());
+ public Collection<RMContainer> getLiveContainers() {
+ try {
+ readLock.lock();
+ return new ArrayList<>(liveContainers.values());
+ } finally {
+ readLock.unlock();
+ }
}
public AppSchedulingInfo getAppSchedulingInfo() {
@@ -243,20 +258,36 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return appSchedulingInfo.getSchedulerKeys();
}
- public synchronized ResourceRequest getResourceRequest(
+ public ResourceRequest getResourceRequest(
SchedulerRequestKey schedulerKey, String resourceName) {
- return appSchedulingInfo.getResourceRequest(schedulerKey, resourceName);
+ try {
+ readLock.lock();
+ return appSchedulingInfo.getResourceRequest(schedulerKey, resourceName);
+ } finally {
+ readLock.unlock();
+ }
+
}
- public synchronized int getTotalRequiredResources(
+ public int getTotalRequiredResources(
SchedulerRequestKey schedulerKey) {
- ResourceRequest request =
- getResourceRequest(schedulerKey, ResourceRequest.ANY);
- return request == null ? 0 : request.getNumContainers();
+ try {
+ readLock.lock();
+ ResourceRequest request =
+ getResourceRequest(schedulerKey, ResourceRequest.ANY);
+ return request == null ? 0 : request.getNumContainers();
+ } finally {
+ readLock.unlock();
+ }
}
- public synchronized Resource getResource(SchedulerRequestKey schedulerKey) {
- return appSchedulingInfo.getResource(schedulerKey);
+ public Resource getResource(SchedulerRequestKey schedulerKey) {
+ try {
+ readLock.lock();
+ return appSchedulingInfo.getResource(schedulerKey);
+ } finally {
+ readLock.unlock();
+ }
}
public String getQueueName() {
@@ -291,38 +322,48 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return unmanagedAM;
}
- public synchronized RMContainer getRMContainer(ContainerId id) {
+ public RMContainer getRMContainer(ContainerId id) {
return liveContainers.get(id);
}
- public synchronized void addRMContainer(
+ public void addRMContainer(
ContainerId id, RMContainer rmContainer) {
- liveContainers.put(id, rmContainer);
- if (rmContainer.isRemotelyAllocated()) {
- this.attemptResourceUsageAllocatedRemotely.incUsed(
- rmContainer.getAllocatedResource());
+ try {
+ writeLock.lock();
+ liveContainers.put(id, rmContainer);
+ if (rmContainer.isRemotelyAllocated()) {
+ this.attemptResourceUsageAllocatedRemotely.incUsed(
+ rmContainer.getAllocatedResource());
+ }
+ } finally {
+ writeLock.unlock();
}
}
- public synchronized void removeRMContainer(ContainerId containerId) {
- RMContainer rmContainer = liveContainers.remove(containerId);
- if (rmContainer != null && rmContainer.isRemotelyAllocated()) {
- this.attemptResourceUsageAllocatedRemotely.decUsed(
- rmContainer.getAllocatedResource());
+ public void removeRMContainer(ContainerId containerId) {
+ try {
+ writeLock.lock();
+ RMContainer rmContainer = liveContainers.remove(containerId);
+ if (rmContainer != null && rmContainer.isRemotelyAllocated()) {
+ this.attemptResourceUsageAllocatedRemotely.decUsed(
+ rmContainer.getAllocatedResource());
+ }
+ } finally {
+ writeLock.unlock();
}
}
- protected synchronized void resetReReservations(
+ protected void resetReReservations(
SchedulerRequestKey schedulerKey) {
reReservations.setCount(schedulerKey, 0);
}
- protected synchronized void addReReservation(
+ protected void addReReservation(
SchedulerRequestKey schedulerKey) {
reReservations.add(schedulerKey);
}
- public synchronized int getReReservations(SchedulerRequestKey schedulerKey) {
+ public int getReReservations(SchedulerRequestKey schedulerKey) {
return reReservations.count(schedulerKey);
}
@@ -333,7 +374,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
*/
@Stable
@Private
- public synchronized Resource getCurrentReservation() {
+ public Resource getCurrentReservation() {
return attemptResourceUsage.getReserved();
}
@@ -341,28 +382,43 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return queue;
}
- public synchronized boolean updateResourceRequests(
+ public boolean updateResourceRequests(
List<ResourceRequest> requests) {
- if (!isStopped) {
- return appSchedulingInfo.updateResourceRequests(requests, false);
+ try {
+ writeLock.lock();
+ if (!isStopped) {
+ return appSchedulingInfo.updateResourceRequests(requests, false);
+ }
+ return false;
+ } finally {
+ writeLock.unlock();
}
- return false;
}
- public synchronized void recoverResourceRequestsForContainer(
+ public void recoverResourceRequestsForContainer(
List<ResourceRequest> requests) {
- if (!isStopped) {
- appSchedulingInfo.updateResourceRequests(requests, true);
+ try {
+ writeLock.lock();
+ if (!isStopped) {
+ appSchedulingInfo.updateResourceRequests(requests, true);
+ }
+ } finally {
+ writeLock.unlock();
}
}
- public synchronized void stop(RMAppAttemptState rmAppAttemptFinalState) {
- // Cleanup all scheduling information
- isStopped = true;
- appSchedulingInfo.stop();
+ public void stop(RMAppAttemptState rmAppAttemptFinalState) {
+ try {
+ writeLock.lock();
+ // Cleanup all scheduling information
+ isStopped = true;
+ appSchedulingInfo.stop();
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized boolean isStopped() {
+ public boolean isStopped() {
return isStopped;
}
@@ -370,29 +426,40 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
* Get the list of reserved containers
* @return All of the reserved containers.
*/
- public synchronized List<RMContainer> getReservedContainers() {
- List<RMContainer> reservedContainers = new ArrayList<RMContainer>();
- for (Map.Entry<SchedulerRequestKey, Map<NodeId, RMContainer>> e :
- this.reservedContainers.entrySet()) {
- reservedContainers.addAll(e.getValue().values());
+ public List<RMContainer> getReservedContainers() {
+ List<RMContainer> list = new ArrayList<>();
+ try {
+ readLock.lock();
+ for (Entry<SchedulerRequestKey, Map<NodeId, RMContainer>> e :
+ this.reservedContainers.entrySet()) {
+ list.addAll(e.getValue().values());
+ }
+ return list;
+ } finally {
+ readLock.unlock();
}
- return reservedContainers;
+
}
- public synchronized boolean reserveIncreasedContainer(SchedulerNode node,
+ public boolean reserveIncreasedContainer(SchedulerNode node,
SchedulerRequestKey schedulerKey, RMContainer rmContainer,
Resource reservedResource) {
- if (commonReserve(node, schedulerKey, rmContainer, reservedResource)) {
- attemptResourceUsage.incReserved(node.getPartition(),
- reservedResource);
- // succeeded
- return true;
+ try {
+ writeLock.lock();
+ if (commonReserve(node, schedulerKey, rmContainer, reservedResource)) {
+ attemptResourceUsage.incReserved(node.getPartition(), reservedResource);
+ // succeeded
+ return true;
+ }
+
+ return false;
+ } finally {
+ writeLock.unlock();
}
-
- return false;
+
}
- private synchronized boolean commonReserve(SchedulerNode node,
+ private boolean commonReserve(SchedulerNode node,
SchedulerRequestKey schedulerKey, RMContainer rmContainer,
Resource reservedResource) {
try {
@@ -423,101 +490,100 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return true;
}
- public synchronized RMContainer reserve(SchedulerNode node,
+ public RMContainer reserve(SchedulerNode node,
SchedulerRequestKey schedulerKey, RMContainer rmContainer,
Container container) {
- // Create RMContainer if necessary
- if (rmContainer == null) {
- rmContainer =
- new RMContainerImpl(container, getApplicationAttemptId(),
- node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
- attemptResourceUsage.incReserved(node.getPartition(),
- container.getResource());
- ((RMContainerImpl)rmContainer).setQueueName(this.getQueueName());
-
- // Reset the re-reservation count
- resetReReservations(schedulerKey);
- } else {
- // Note down the re-reservation
- addReReservation(schedulerKey);
- }
-
- commonReserve(node, schedulerKey, rmContainer, container.getResource());
+ try {
+ writeLock.lock();
+ // Create RMContainer if necessary
+ if (rmContainer == null) {
+ rmContainer = new RMContainerImpl(container, getApplicationAttemptId(),
+ node.getNodeID(), appSchedulingInfo.getUser(), rmContext);
+ attemptResourceUsage.incReserved(node.getPartition(),
+ container.getResource());
+ ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
+
+ // Reset the re-reservation count
+ resetReReservations(schedulerKey);
+ } else{
+ // Note down the re-reservation
+ addReReservation(schedulerKey);
+ }
- return rmContainer;
- }
-
- /**
- * Has the application reserved the given <code>node</code> at the
- * given <code>priority</code>?
- * @param node node to be checked
- * @param schedulerKey scheduler key of reserved container
- * @return true is reserved, false if not
- */
- public synchronized boolean isReserved(SchedulerNode node,
- SchedulerRequestKey schedulerKey) {
- Map<NodeId, RMContainer> reservedContainers =
- this.reservedContainers.get(schedulerKey);
- if (reservedContainers != null) {
- return reservedContainers.containsKey(node.getNodeID());
+ commonReserve(node, schedulerKey, rmContainer, container.getResource());
+
+ return rmContainer;
+ } finally {
+ writeLock.unlock();
}
- return false;
+
}
-
- public synchronized void setHeadroom(Resource globalLimit) {
- this.resourceLimit = globalLimit;
+
+ public void setHeadroom(Resource globalLimit) {
+ this.resourceLimit = Resources.componentwiseMax(globalLimit,
+ Resources.none());
}
/**
* Get available headroom in terms of resources for the application's user.
* @return available resource headroom
*/
- public synchronized Resource getHeadroom() {
- // Corner case to deal with applications being slightly over-limit
- if (resourceLimit.getMemorySize() < 0) {
- resourceLimit.setMemorySize(0);
- }
-
+ public Resource getHeadroom() {
return resourceLimit;
}
- public synchronized int getNumReservedContainers(
+ public int getNumReservedContainers(
SchedulerRequestKey schedulerKey) {
- Map<NodeId, RMContainer> reservedContainers =
- this.reservedContainers.get(schedulerKey);
- return (reservedContainers == null) ? 0 : reservedContainers.size();
+ try {
+ readLock.lock();
+ Map<NodeId, RMContainer> map = this.reservedContainers.get(
+ schedulerKey);
+ return (map == null) ? 0 : map.size();
+ } finally {
+ readLock.unlock();
+ }
}
@SuppressWarnings("unchecked")
- public synchronized void containerLaunchedOnNode(ContainerId containerId,
+ public void containerLaunchedOnNode(ContainerId containerId,
NodeId nodeId) {
- // Inform the container
- RMContainer rmContainer = getRMContainer(containerId);
- if (rmContainer == null) {
- // Some unknown container sneaked into the system. Kill it.
- rmContext.getDispatcher().getEventHandler()
- .handle(new RMNodeCleanContainerEvent(nodeId, containerId));
- return;
- }
+ try {
+ writeLock.lock();
+ // Inform the container
+ RMContainer rmContainer = getRMContainer(containerId);
+ if (rmContainer == null) {
+ // Some unknown container sneaked into the system. Kill it.
+ rmContext.getDispatcher().getEventHandler().handle(
+ new RMNodeCleanContainerEvent(nodeId, containerId));
+ return;
+ }
- rmContainer.handle(new RMContainerEvent(containerId,
- RMContainerEventType.LAUNCHED));
+ rmContainer.handle(
+ new RMContainerEvent(containerId, RMContainerEventType.LAUNCHED));
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized void showRequests() {
+ public void showRequests() {
if (LOG.isDebugEnabled()) {
- for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) {
- Map<String, ResourceRequest> requests =
- getResourceRequests(schedulerKey);
- if (requests != null) {
- LOG.debug("showRequests:" + " application=" + getApplicationId()
- + " headRoom=" + getHeadroom() + " currentConsumption="
- + attemptResourceUsage.getUsed().getMemorySize());
- for (ResourceRequest request : requests.values()) {
+ try {
+ readLock.lock();
+ for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) {
+ Map<String, ResourceRequest> requests = getResourceRequests(
+ schedulerKey);
+ if (requests != null) {
LOG.debug("showRequests:" + " application=" + getApplicationId()
- + " request=" + request);
+ + " headRoom=" + getHeadroom() + " currentConsumption="
+ + attemptResourceUsage.getUsed().getMemorySize());
+ for (ResourceRequest request : requests.values()) {
+ LOG.debug("showRequests:" + " application=" + getApplicationId()
+ + " request=" + request);
+ }
}
}
+ } finally {
+ readLock.unlock();
}
}
}
@@ -572,54 +638,75 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// Create container token and update NMToken altogether, if either of them fails for
// some reason like DNS unavailable, do not return this container and keep it
// in the newlyAllocatedContainers waiting to be refetched.
- public synchronized List<Container> pullNewlyAllocatedContainers() {
- List<Container> returnContainerList =
- new ArrayList<Container>(newlyAllocatedContainers.size());
- for (Iterator<RMContainer> i = newlyAllocatedContainers.iterator(); i
- .hasNext();) {
- RMContainer rmContainer = i.next();
- Container updatedContainer =
- updateContainerAndNMToken(rmContainer, true, false);
- // Only add container to return list when it's not null. updatedContainer
- // could be null when generate token failed, it can be caused by DNS
- // resolving failed.
- if (updatedContainer != null) {
- returnContainerList.add(updatedContainer);
- i.remove();
+ public List<Container> pullNewlyAllocatedContainers() {
+ try {
+ writeLock.lock();
+ List<Container> returnContainerList = new ArrayList<Container>(
+ newlyAllocatedContainers.size());
+
+ Iterator<RMContainer> i = newlyAllocatedContainers.iterator();
+ while (i.hasNext()) {
+ RMContainer rmContainer = i.next();
+ Container updatedContainer = updateContainerAndNMToken(rmContainer,
+ true, false);
+ // Only add container to return list when it's not null.
+ // updatedContainer could be null when generate token failed, it can be
+ // caused by DNS resolving failed.
+ if (updatedContainer != null) {
+ returnContainerList.add(updatedContainer);
+ i.remove();
+ }
}
+ return returnContainerList;
+ } finally {
+ writeLock.unlock();
}
- return returnContainerList;
+
}
- private synchronized List<Container> pullNewlyUpdatedContainers(
+ private List<Container> pullNewlyUpdatedContainers(
Map<ContainerId, RMContainer> updatedContainerMap, boolean increase) {
- List<Container> returnContainerList =
- new ArrayList<Container>(updatedContainerMap.size());
- for (Iterator<Entry<ContainerId, RMContainer>> i =
- updatedContainerMap.entrySet().iterator(); i.hasNext();) {
- RMContainer rmContainer = i.next().getValue();
- Container updatedContainer =
- updateContainerAndNMToken(rmContainer, false, increase);
- if (updatedContainer != null) {
- returnContainerList.add(updatedContainer);
- i.remove();
+ try {
+ writeLock.lock();
+ List <Container> returnContainerList = new ArrayList <Container>(
+ updatedContainerMap.size());
+
+ Iterator<Entry<ContainerId, RMContainer>> i =
+ updatedContainerMap.entrySet().iterator();
+ while (i.hasNext()) {
+ RMContainer rmContainer = i.next().getValue();
+ Container updatedContainer = updateContainerAndNMToken(rmContainer,
+ false, increase);
+ if (updatedContainer != null) {
+ returnContainerList.add(updatedContainer);
+ i.remove();
+ }
}
+ return returnContainerList;
+ } finally {
+ writeLock.unlock();
}
- return returnContainerList;
+
}
- public synchronized List<Container> pullNewlyIncreasedContainers() {
+ public List<Container> pullNewlyIncreasedContainers() {
return pullNewlyUpdatedContainers(newlyIncreasedContainers, true);
}
- public synchronized List<Container> pullNewlyDecreasedContainers() {
+ public List<Container> pullNewlyDecreasedContainers() {
return pullNewlyUpdatedContainers(newlyDecreasedContainers, false);
}
- public synchronized List<NMToken> pullUpdatedNMTokens() {
- List<NMToken> returnList = new ArrayList<NMToken>(updatedNMTokens);
- updatedNMTokens.clear();
- return returnList;
+ public List<NMToken> pullUpdatedNMTokens() {
+ try {
+ writeLock.lock();
+ List <NMToken> returnList = new ArrayList<>(updatedNMTokens);
+ updatedNMTokens.clear();
+ return returnList;
+ } finally {
+ writeLock.unlock();
+ }
+
}
public boolean isWaitingForAMContainer() {
@@ -628,53 +715,63 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return (!unmanagedAM && appAttempt.getMasterContainer() == null);
}
- public synchronized void updateBlacklist(List<String> blacklistAdditions,
+ public void updateBlacklist(List<String> blacklistAdditions,
List<String> blacklistRemovals) {
- if (!isStopped) {
- if (isWaitingForAMContainer()) {
- // The request is for the AM-container, and the AM-container is launched
- // by the system. So, update the places that are blacklisted by system
- // (as opposed to those blacklisted by the application).
- this.appSchedulingInfo.updatePlacesBlacklistedBySystem(
- blacklistAdditions, blacklistRemovals);
- } else {
- this.appSchedulingInfo.updatePlacesBlacklistedByApp(blacklistAdditions,
- blacklistRemovals);
+ try {
+ writeLock.lock();
+ if (!isStopped) {
+ if (isWaitingForAMContainer()) {
+ // The request is for the AM-container, and the AM-container is
+ // launched by the system. So, update the places that are blacklisted
+ // by system (as opposed to those blacklisted by the application).
+ this.appSchedulingInfo.updatePlacesBlacklistedBySystem(
+ blacklistAdditions, blacklistRemovals);
+ } else{
+ this.appSchedulingInfo.updatePlacesBlacklistedByApp(
+ blacklistAdditions, blacklistRemovals);
+ }
}
+ } finally {
+ writeLock.unlock();
}
}
public boolean isPlaceBlacklisted(String resourceName) {
- boolean forAMContainer = isWaitingForAMContainer();
- return this.appSchedulingInfo.isPlaceBlacklisted(resourceName,
- forAMContainer);
+ try {
+ readLock.lock();
+ boolean forAMContainer = isWaitingForAMContainer();
+ return this.appSchedulingInfo.isPlaceBlacklisted(resourceName,
+ forAMContainer);
+ } finally {
+ readLock.unlock();
+ }
}
- public synchronized int addMissedNonPartitionedRequestSchedulingOpportunity(
+ public int addMissedNonPartitionedRequestSchedulingOpportunity(
SchedulerRequestKey schedulerKey) {
- missedNonPartitionedReqSchedulingOpportunity.add(schedulerKey);
- return missedNonPartitionedReqSchedulingOpportunity.count(schedulerKey);
+ return missedNonPartitionedReqSchedulingOpportunity.add(
+ schedulerKey, 1) + 1;
}
- public synchronized void
+ public void
resetMissedNonPartitionedRequestSchedulingOpportunity(
SchedulerRequestKey schedulerKey) {
missedNonPartitionedReqSchedulingOpportunity.setCount(schedulerKey, 0);
}
- public synchronized void addSchedulingOpportunity(
+ public void addSchedulingOpportunity(
SchedulerRequestKey schedulerKey) {
- int count = schedulingOpportunities.count(schedulerKey);
- if (count < Integer.MAX_VALUE) {
- schedulingOpportunities.setCount(schedulerKey, count + 1);
+ try {
+ schedulingOpportunities.add(schedulerKey, 1);
+ } catch (IllegalArgumentException e) {
+ // This happens when count = MAX_INT, ignore the exception
}
}
- public synchronized void subtractSchedulingOpportunity(
+ public void subtractSchedulingOpportunity(
SchedulerRequestKey schedulerKey) {
- int count = schedulingOpportunities.count(schedulerKey) - 1;
- this.schedulingOpportunities.setCount(schedulerKey, Math.max(count, 0));
+ this.schedulingOpportunities.removeExactly(schedulerKey, 1);
}
/**
@@ -684,7 +781,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
* @param schedulerKey Scheduler Key
* @return number of scheduling opportunities
*/
- public synchronized int getSchedulingOpportunities(
+ public int getSchedulingOpportunities(
SchedulerRequestKey schedulerKey) {
return schedulingOpportunities.count(schedulerKey);
}
@@ -696,16 +793,22 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
*
* @param schedulerKey The priority of the container scheduled.
*/
- public synchronized void resetSchedulingOpportunities(
+ public void resetSchedulingOpportunities(
SchedulerRequestKey schedulerKey) {
resetSchedulingOpportunities(schedulerKey, System.currentTimeMillis());
}
// used for continuous scheduling
- public synchronized void resetSchedulingOpportunities(
+ public void resetSchedulingOpportunities(
SchedulerRequestKey schedulerKey, long currentTimeMs) {
- lastScheduledContainer.put(schedulerKey, currentTimeMs);
- schedulingOpportunities.setCount(schedulerKey, 0);
+ try {
+ writeLock.lock();
+ lastScheduledContainer.put(schedulerKey, currentTimeMs);
+ schedulingOpportunities.setCount(schedulerKey, 0);
+ } finally {
+ writeLock.unlock();
+ }
+
}
@VisibleForTesting
@@ -713,7 +816,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
schedulingOpportunities.setCount(schedulerKey, count);
}
- synchronized AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
+ private AggregateAppResourceUsage getRunningAggregateAppResourceUsage() {
long currentTimeMillis = System.currentTimeMillis();
// Don't walk the whole container list if the resources were computed
// recently.
@@ -737,101 +840,120 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return new AggregateAppResourceUsage(lastMemorySeconds, lastVcoreSeconds);
}
- public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
- AggregateAppResourceUsage runningResourceUsage =
- getRunningAggregateAppResourceUsage();
- Resource usedResourceClone =
- Resources.clone(attemptResourceUsage.getAllUsed());
- Resource reservedResourceClone =
- Resources.clone(attemptResourceUsage.getReserved());
- Resource cluster = rmContext.getScheduler().getClusterResource();
- ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
- float queueUsagePerc = 0.0f;
- float clusterUsagePerc = 0.0f;
- if (!calc.isInvalidDivisor(cluster)) {
- queueUsagePerc =
- calc.divide(cluster, usedResourceClone, Resources.multiply(cluster,
- queue.getQueueInfo(false, false).getCapacity())) * 100;
- clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster) * 100;
+ public ApplicationResourceUsageReport getResourceUsageReport() {
+ try {
+ writeLock.lock();
+ AggregateAppResourceUsage runningResourceUsage =
+ getRunningAggregateAppResourceUsage();
+ Resource usedResourceClone = Resources.clone(
+ attemptResourceUsage.getAllUsed());
+ Resource reservedResourceClone = Resources.clone(
+ attemptResourceUsage.getReserved());
+ Resource cluster = rmContext.getScheduler().getClusterResource();
+ ResourceCalculator calc =
+ rmContext.getScheduler().getResourceCalculator();
+ float queueUsagePerc = 0.0f;
+ float clusterUsagePerc = 0.0f;
+ if (!calc.isInvalidDivisor(cluster)) {
+ queueUsagePerc = calc.divide(cluster, usedResourceClone, Resources
+ .multiply(cluster, queue.getQueueInfo(false, false).getCapacity()))
+ * 100;
+ clusterUsagePerc = calc.divide(cluster, usedResourceClone, cluster)
+ * 100;
+ }
+ return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
+ reservedContainers.size(), usedResourceClone, reservedResourceClone,
+ Resources.add(usedResourceClone, reservedResourceClone),
+ runningResourceUsage.getMemorySeconds(),
+ runningResourceUsage.getVcoreSeconds(), queueUsagePerc,
+ clusterUsagePerc);
+ } finally {
+ writeLock.unlock();
}
- return ApplicationResourceUsageReport.newInstance(liveContainers.size(),
- reservedContainers.size(), usedResourceClone, reservedResourceClone,
- Resources.add(usedResourceClone, reservedResourceClone),
- runningResourceUsage.getMemorySeconds(),
- runningResourceUsage.getVcoreSeconds(), queueUsagePerc,
- clusterUsagePerc);
}
- public synchronized Map<ContainerId, RMContainer> getLiveContainersMap() {
+ @VisibleForTesting
+ public Map<ContainerId, RMContainer> getLiveContainersMap() {
return this.liveContainers;
}
- public synchronized Resource getResourceLimit() {
- return this.resourceLimit;
- }
-
- public synchronized Map<SchedulerRequestKey, Long>
+ public Map<SchedulerRequestKey, Long>
getLastScheduledContainer() {
return this.lastScheduledContainer;
}
- public synchronized void transferStateFromPreviousAttempt(
+ public void transferStateFromPreviousAttempt(
SchedulerApplicationAttempt appAttempt) {
- this.liveContainers = appAttempt.getLiveContainersMap();
- // this.reReservations = appAttempt.reReservations;
- this.attemptResourceUsage.copyAllUsed(appAttempt.attemptResourceUsage);
- this.resourceLimit = appAttempt.getResourceLimit();
- // this.currentReservation = appAttempt.currentReservation;
- // this.newlyAllocatedContainers = appAttempt.newlyAllocatedContainers;
- // this.schedulingOpportunities = appAttempt.schedulingOpportunities;
- this.lastScheduledContainer = appAttempt.getLastScheduledContainer();
- this.appSchedulingInfo
- .transferStateFromPreviousAppSchedulingInfo(appAttempt.appSchedulingInfo);
+ try {
+ writeLock.lock();
+ this.liveContainers = appAttempt.getLiveContainersMap();
+ // this.reReservations = appAttempt.reReservations;
+ this.attemptResourceUsage.copyAllUsed(appAttempt.attemptResourceUsage);
+ this.setHeadroom(appAttempt.resourceLimit);
+ // this.currentReservation = appAttempt.currentReservation;
+ // this.newlyAllocatedContainers = appAttempt.newlyAllocatedContainers;
+ // this.schedulingOpportunities = appAttempt.schedulingOpportunities;
+ this.lastScheduledContainer = appAttempt.getLastScheduledContainer();
+ this.appSchedulingInfo.transferStateFromPreviousAppSchedulingInfo(
+ appAttempt.appSchedulingInfo);
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized void move(Queue newQueue) {
- QueueMetrics oldMetrics = queue.getMetrics();
- QueueMetrics newMetrics = newQueue.getMetrics();
- String newQueueName = newQueue.getQueueName();
- String user = getUser();
- for (RMContainer liveContainer : liveContainers.values()) {
- Resource resource = liveContainer.getContainer().getResource();
- ((RMContainerImpl)liveContainer).setQueueName(newQueueName);
- oldMetrics.releaseResources(user, 1, resource);
- newMetrics.allocateResources(user, 1, resource, false);
- }
- for (Map<NodeId, RMContainer> map : reservedContainers.values()) {
- for (RMContainer reservedContainer : map.values()) {
- ((RMContainerImpl)reservedContainer).setQueueName(newQueueName);
- Resource resource = reservedContainer.getReservedResource();
- oldMetrics.unreserveResource(user, resource);
- newMetrics.reserveResource(user, resource);
+ public void move(Queue newQueue) {
+ try {
+ writeLock.lock();
+ QueueMetrics oldMetrics = queue.getMetrics();
+ QueueMetrics newMetrics = newQueue.getMetrics();
+ String newQueueName = newQueue.getQueueName();
+ String user = getUser();
+ for (RMContainer liveContainer : liveContainers.values()) {
+ Resource resource = liveContainer.getContainer().getResource();
+ ((RMContainerImpl) liveContainer).setQueueName(newQueueName);
+ oldMetrics.releaseResources(user, 1, resource);
+ newMetrics.allocateResources(user, 1, resource, false);
+ }
+ for (Map<NodeId, RMContainer> map : reservedContainers.values()) {
+ for (RMContainer reservedContainer : map.values()) {
+ ((RMContainerImpl) reservedContainer).setQueueName(newQueueName);
+ Resource resource = reservedContainer.getReservedResource();
+ oldMetrics.unreserveResource(user, resource);
+ newMetrics.reserveResource(user, resource);
+ }
}
- }
- appSchedulingInfo.move(newQueue);
- this.queue = newQueue;
+ appSchedulingInfo.move(newQueue);
+ this.queue = newQueue;
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized void recoverContainer(SchedulerNode node,
+ public void recoverContainer(SchedulerNode node,
RMContainer rmContainer) {
- // recover app scheduling info
- appSchedulingInfo.recoverContainer(rmContainer);
+ try {
+ writeLock.lock();
+ // recover app scheduling info
+ appSchedulingInfo.recoverContainer(rmContainer);
- if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
- return;
+ if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
+ return;
+ }
+ LOG.info("SchedulerAttempt " + getApplicationAttemptId()
+ + " is recovering container " + rmContainer.getContainerId());
+ liveContainers.put(rmContainer.getContainerId(), rmContainer);
+ attemptResourceUsage.incUsed(node.getPartition(),
+ rmContainer.getContainer().getResource());
+
+ // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
+ // is called.
+ // newlyAllocatedContainers.add(rmContainer);
+ // schedulingOpportunities
+ // lastScheduledContainer
+ } finally {
+ writeLock.unlock();
}
- LOG.info("SchedulerAttempt " + getApplicationAttemptId()
- + " is recovering container " + rmContainer.getContainerId());
- liveContainers.put(rmContainer.getContainerId(), rmContainer);
- attemptResourceUsage.incUsed(node.getPartition(), rmContainer
- .getContainer().getResource());
-
- // resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
- // is called.
- // newlyAllocatedContainers.add(rmContainer);
- // schedulingOpportunities
- // lastScheduledContainer
}
public void incNumAllocatedContainers(NodeType containerType,
@@ -915,49 +1037,64 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return attemptResourceUsage;
}
- public synchronized boolean removeIncreaseRequest(NodeId nodeId,
+ public boolean removeIncreaseRequest(NodeId nodeId,
SchedulerRequestKey schedulerKey, ContainerId containerId) {
- return appSchedulingInfo.removeIncreaseRequest(nodeId, schedulerKey,
- containerId);
+ try {
+ writeLock.lock();
+ return appSchedulingInfo.removeIncreaseRequest(nodeId, schedulerKey,
+ containerId);
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized boolean updateIncreaseRequests(
+ public boolean updateIncreaseRequests(
List<SchedContainerChangeRequest> increaseRequests) {
- return appSchedulingInfo.updateIncreaseRequests(increaseRequests);
+ try {
+ writeLock.lock();
+ return appSchedulingInfo.updateIncreaseRequests(increaseRequests);
+ } finally {
+ writeLock.unlock();
+ }
}
- private synchronized void changeContainerResource(
+ private void changeContainerResource(
SchedContainerChangeRequest changeRequest, boolean increase) {
- if (increase) {
- appSchedulingInfo.increaseContainer(changeRequest);
- } else {
- appSchedulingInfo.decreaseContainer(changeRequest);
- }
+ try {
+ writeLock.lock();
+ if (increase) {
+ appSchedulingInfo.increaseContainer(changeRequest);
+ } else{
+ appSchedulingInfo.decreaseContainer(changeRequest);
+ }
- RMContainer changedRMContainer = changeRequest.getRMContainer();
- changedRMContainer.handle(
- new RMContainerChangeResourceEvent(changeRequest.getContainerId(),
- changeRequest.getTargetCapacity(), increase));
-
- // remove pending and not pulled by AM newly-increased/decreased-containers
- // and add the new one
- if (increase) {
- newlyDecreasedContainers.remove(changeRequest.getContainerId());
- newlyIncreasedContainers.put(changeRequest.getContainerId(),
- changedRMContainer);
- } else {
- newlyIncreasedContainers.remove(changeRequest.getContainerId());
- newlyDecreasedContainers.put(changeRequest.getContainerId(),
- changedRMContainer);
+ RMContainer changedRMContainer = changeRequest.getRMContainer();
+ changedRMContainer.handle(
+ new RMContainerChangeResourceEvent(changeRequest.getContainerId(),
+ changeRequest.getTargetCapacity(), increase));
+
+ // remove pending and not pulled by AM newly-increased or
+ // decreased-containers and add the new one
+ if (increase) {
+ newlyDecreasedContainers.remove(changeRequest.getContainerId());
+ newlyIncreasedContainers.put(changeRequest.getContainerId(),
+ changedRMContainer);
+ } else{
+ newlyIncreasedContainers.remove(changeRequest.getContainerId());
+ newlyDecreasedContainers.put(changeRequest.getContainerId(),
+ changedRMContainer);
+ }
+ } finally {
+ writeLock.unlock();
}
}
- public synchronized void decreaseContainer(
+ public void decreaseContainer(
SchedContainerChangeRequest decreaseRequest) {
changeContainerResource(decreaseRequest, false);
}
- public synchronized void increaseContainer(
+ public void increaseContainer(
SchedContainerChangeRequest increaseRequest) {
changeContainerResource(increaseRequest, true);
}
@@ -1025,7 +1162,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
this.isAttemptRecovering = isRecovering;
}
- public static enum AMState {
+ /**
+ * Different state for Application Master, user can see this state from web UI
+ */
+ public enum AMState {
UNMANAGED("User launched the Application Master, since it's unmanaged. "),
INACTIVATED("Application is added to the scheduler and is not yet activated. "),
ACTIVATED("Application is Activated, waiting for resources to be assigned for AM. "),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a30f2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 8d4042c..1a3f71f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -251,7 +251,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
return result;
}
- public synchronized float getLocalityWaitFactor(
+ public float getLocalityWaitFactor(
SchedulerRequestKey schedulerKey, int clusterNodes) {
// Estimate: Required unique resources (i.e. hosts + racks)
int requiredResources =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a30f2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 9c84a23..f40ecd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -63,7 +63,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMCont
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
@@ -99,7 +98,6 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
* to hold the message if its app doesn't not get container from a node
*/
private String appSkipNodeDiagnostics;
- private CapacitySchedulerContext capacitySchedulerContext;
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
@@ -153,118 +151,128 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
containerAllocator = new ContainerAllocator(this, rc, rmContext,
activitiesManager);
-
- if (scheduler instanceof CapacityScheduler) {
- capacitySchedulerContext = (CapacitySchedulerContext) scheduler;
- }
}
- public synchronized boolean containerCompleted(RMContainer rmContainer,
+ public boolean containerCompleted(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event,
String partition) {
- ContainerId containerId = rmContainer.getContainerId();
+ try {
+ writeLock.lock();
+ ContainerId containerId = rmContainer.getContainerId();
- // Remove from the list of containers
- if (null == liveContainers.remove(containerId)) {
- return false;
- }
+ // Remove from the list of containers
+ if (null == liveContainers.remove(containerId)) {
+ return false;
+ }
- // Remove from the list of newly allocated containers if found
- newlyAllocatedContainers.remove(rmContainer);
+ // Remove from the list of newly allocated containers if found
+ newlyAllocatedContainers.remove(rmContainer);
- // Inform the container
- rmContainer.handle(
- new RMContainerFinishedEvent(containerId, containerStatus, event));
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerFinishedEvent(containerId, containerStatus, event));
- containersToPreempt.remove(containerId);
+ containersToPreempt.remove(containerId);
- Resource containerResource = rmContainer.getContainer().getResource();
- RMAuditLogger.logSuccess(getUser(),
- AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId, containerResource);
-
- // Update usage metrics
- queue.getMetrics().releaseResources(getUser(), 1, containerResource);
- attemptResourceUsage.decUsed(partition, containerResource);
+ Resource containerResource = rmContainer.getContainer().getResource();
+ RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
+ "SchedulerApp", getApplicationId(), containerId, containerResource);
+
+ // Update usage metrics
+ queue.getMetrics().releaseResources(getUser(), 1, containerResource);
+ attemptResourceUsage.decUsed(partition, containerResource);
- // Clear resource utilization metrics cache.
- lastMemoryAggregateAllocationUpdateTime = -1;
+ // Clear resource utilization metrics cache.
+ lastMemoryAggregateAllocationUpdateTime = -1;
- return true;
+ return true;
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized RMContainer allocate(NodeType type, FiCaSchedulerNode node,
+ public RMContainer allocate(NodeType type, FiCaSchedulerNode node,
SchedulerRequestKey schedulerKey, ResourceRequest request,
Container container) {
+ try {
+ writeLock.lock();
- if (isStopped) {
- return null;
- }
-
- // Required sanity check - AM can call 'allocate' to update resource
- // request without locking the scheduler, hence we need to check
- if (getTotalRequiredResources(schedulerKey) <= 0) {
- return null;
- }
+ if (isStopped) {
+ return null;
+ }
+
+ // Required sanity check - AM can call 'allocate' to update resource
+ // request without locking the scheduler, hence we need to check
+ if (getTotalRequiredResources(schedulerKey) <= 0) {
+ return null;
+ }
- // Create RMContainer
- RMContainer rmContainer =
- new RMContainerImpl(container, this.getApplicationAttemptId(),
- node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext,
- request.getNodeLabelExpression());
- ((RMContainerImpl)rmContainer).setQueueName(this.getQueueName());
+ // Create RMContainer
+ RMContainer rmContainer = new RMContainerImpl(container,
+ this.getApplicationAttemptId(), node.getNodeID(),
+ appSchedulingInfo.getUser(), this.rmContext,
+ request.getNodeLabelExpression());
+ ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
- updateAMContainerDiagnostics(AMState.ASSIGNED, null);
+ updateAMContainerDiagnostics(AMState.ASSIGNED, null);
- // Add it to allContainers list.
- newlyAllocatedContainers.add(rmContainer);
+ // Add it to allContainers list.
+ newlyAllocatedContainers.add(rmContainer);
- ContainerId containerId = container.getId();
- liveContainers.put(containerId, rmContainer);
+ ContainerId containerId = container.getId();
+ liveContainers.put(containerId, rmContainer);
- // Update consumption and track allocations
- List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
- type, node, schedulerKey, request, container);
+ // Update consumption and track allocations
+ List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
+ type, node, schedulerKey, request, container);
- attemptResourceUsage.incUsed(node.getPartition(), container.getResource());
+ attemptResourceUsage.incUsed(node.getPartition(),
+ container.getResource());
- // Update resource requests related to "request" and store in RMContainer
- ((RMContainerImpl)rmContainer).setResourceRequests(resourceRequestList);
+ // Update resource requests related to "request" and store in RMContainer
+ ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
- // Inform the container
- rmContainer.handle(
- new RMContainerEvent(containerId, RMContainerEventType.START));
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerEvent(containerId, RMContainerEventType.START));
- if (LOG.isDebugEnabled()) {
- LOG.debug("allocate: applicationAttemptId="
- + containerId.getApplicationAttemptId()
- + " container=" + containerId + " host="
- + container.getNodeId().getHost() + " type=" + type);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("allocate: applicationAttemptId=" + containerId
+ .getApplicationAttemptId() + " container=" + containerId + " host="
+ + container.getNodeId().getHost() + " type=" + type);
+ }
+ RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
+ "SchedulerApp", getApplicationId(), containerId,
+ container.getResource());
+
+ return rmContainer;
+ } finally {
+ writeLock.unlock();
}
- RMAuditLogger.logSuccess(getUser(),
- AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId, container.getResource());
-
- return rmContainer;
}
- public synchronized boolean unreserve(SchedulerRequestKey schedulerKey,
+ public boolean unreserve(SchedulerRequestKey schedulerKey,
FiCaSchedulerNode node, RMContainer rmContainer) {
- // Cancel increase request (if it has reserved increase request
- rmContainer.cancelIncreaseReservation();
-
- // Done with the reservation?
- if (internalUnreserve(node, schedulerKey)) {
- node.unreserveResource(this);
-
- // Update reserved metrics
- queue.getMetrics().unreserveResource(getUser(),
- rmContainer.getReservedResource());
- queue.decReservedResource(node.getPartition(),
- rmContainer.getReservedResource());
- return true;
+ try {
+ writeLock.lock();
+ // Cancel increase request (if it has reserved increase request
+ rmContainer.cancelIncreaseReservation();
+
+ // Done with the reservation?
+ if (internalUnreserve(node, schedulerKey)) {
+ node.unreserveResource(this);
+
+ // Update reserved metrics
+ queue.getMetrics().unreserveResource(getUser(),
+ rmContainer.getReservedResource());
+ queue.decReservedResource(node.getPartition(),
+ rmContainer.getReservedResource());
+ return true;
+ }
+ return false;
+ } finally {
+ writeLock.unlock();
}
- return false;
}
private boolean internalUnreserve(FiCaSchedulerNode node,
@@ -303,33 +311,15 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return false;
}
- public synchronized float getLocalityWaitFactor(
- SchedulerRequestKey schedulerKey, int clusterNodes) {
- // Estimate: Required unique resources (i.e. hosts + racks)
- int requiredResources =
- Math.max(this.getResourceRequests(schedulerKey).size() - 1, 0);
-
- // waitFactor can't be more than '1'
- // i.e. no point skipping more than clustersize opportunities
- return Math.min(((float)requiredResources / clusterNodes), 1.0f);
- }
-
- public synchronized Resource getTotalPendingRequests() {
- Resource ret = Resource.newInstance(0, 0);
- for (ResourceRequest rr : appSchedulingInfo.getAllResourceRequests()) {
- // to avoid double counting we count only "ANY" resource requests
- if (ResourceRequest.isAnyLocation(rr.getResourceName())){
- Resources.addTo(ret,
- Resources.multiply(rr.getCapability(), rr.getNumContainers()));
+ public void markContainerForPreemption(ContainerId cont) {
+ try {
+ writeLock.lock();
+ // ignore already completed containers
+ if (liveContainers.containsKey(cont)) {
+ containersToPreempt.add(cont);
}
- }
- return ret;
- }
-
- public synchronized void markContainerForPreemption(ContainerId cont) {
- // ignore already completed containers
- if (liveContainers.containsKey(cont)) {
- containersToPreempt.add(cont);
+ } finally {
+ writeLock.unlock();
}
}
@@ -343,94 +333,115 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
* @param minimumAllocation
* @return an allocation
*/
- public synchronized Allocation getAllocation(ResourceCalculator rc,
+ public Allocation getAllocation(ResourceCalculator resourceCalculator,
Resource clusterResource, Resource minimumAllocation) {
-
- Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
- new HashSet<ContainerId>(containersToPreempt));
- containersToPreempt.clear();
- Resource tot = Resource.newInstance(0, 0);
- for(ContainerId c : currentContPreemption){
- Resources.addTo(tot,
- liveContainers.get(c).getContainer().getResource());
+ try {
+ writeLock.lock();
+ Set<ContainerId> currentContPreemption = Collections.unmodifiableSet(
+ new HashSet<ContainerId>(containersToPreempt));
+ containersToPreempt.clear();
+ Resource tot = Resource.newInstance(0, 0);
+ for (ContainerId c : currentContPreemption) {
+ Resources.addTo(tot, liveContainers.get(c).getContainer()
+ .getResource());
+ }
+ int numCont = (int) Math.ceil(
+ Resources.divide(rc, clusterResource, tot, minimumAllocation));
+ ResourceRequest rr = ResourceRequest.newInstance(Priority.UNDEFINED,
+ ResourceRequest.ANY, minimumAllocation, numCont);
+ List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
+ List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
+ List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
+ List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
+ Resource headroom = getHeadroom();
+ setApplicationHeadroomForMetrics(headroom);
+ return new Allocation(newlyAllocatedContainers, headroom, null,
+ currentContPreemption, Collections.singletonList(rr), updatedNMTokens,
+ newlyIncreasedContainers, newlyDecreasedContainers);
+ } finally {
+ writeLock.unlock();
}
- int numCont = (int) Math.ceil(
- Resources.divide(rc, clusterResource, tot, minimumAllocation));
- ResourceRequest rr = ResourceRequest.newInstance(
- Priority.UNDEFINED, ResourceRequest.ANY,
- minimumAllocation, numCont);
- List<Container> newlyAllocatedContainers = pullNewlyAllocatedContainers();
- List<Container> newlyIncreasedContainers = pullNewlyIncreasedContainers();
- List<Container> newlyDecreasedContainers = pullNewlyDecreasedContainers();
- List<NMToken> updatedNMTokens = pullUpdatedNMTokens();
- Resource headroom = getHeadroom();
- setApplicationHeadroomForMetrics(headroom);
- return new Allocation(newlyAllocatedContainers, headroom, null,
- currentContPreemption, Collections.singletonList(rr), updatedNMTokens,
- newlyIncreasedContainers, newlyDecreasedContainers);
}
-
- synchronized public NodeId getNodeIdToUnreserve(
+
+ @VisibleForTesting
+ public NodeId getNodeIdToUnreserve(
SchedulerRequestKey schedulerKey, Resource resourceNeedUnreserve,
ResourceCalculator rc, Resource clusterResource) {
+ try {
+ writeLock.lock();
+ // first go around make this algorithm simple and just grab first
+ // reservation that has enough resources
+ Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(
+ schedulerKey);
+
+ if ((reservedContainers != null) && (!reservedContainers.isEmpty())) {
+ for (Map.Entry<NodeId, RMContainer> entry : reservedContainers
+ .entrySet()) {
+ NodeId nodeId = entry.getKey();
+ RMContainer reservedContainer = entry.getValue();
+ if (reservedContainer.hasIncreaseReservation()) {
+ // Currently, only regular container allocation supports continuous
+ // reservation looking, we don't support canceling increase request
+ // reservation when allocating regular container.
+ continue;
+ }
- // first go around make this algorithm simple and just grab first
- // reservation that has enough resources
- Map<NodeId, RMContainer> reservedContainers = this.reservedContainers
- .get(schedulerKey);
-
- if ((reservedContainers != null) && (!reservedContainers.isEmpty())) {
- for (Map.Entry<NodeId, RMContainer> entry : reservedContainers.entrySet()) {
- NodeId nodeId = entry.getKey();
- RMContainer reservedContainer = entry.getValue();
- if (reservedContainer.hasIncreaseReservation()) {
- // Currently, only regular container allocation supports continuous
- // reservation looking, we don't support canceling increase request
- // reservation when allocating regular container.
- continue;
- }
-
- Resource reservedResource = reservedContainer.getReservedResource();
-
- // make sure we unreserve one with at least the same amount of
- // resources, otherwise could affect capacity limits
- if (Resources.fitsIn(rc, clusterResource, resourceNeedUnreserve,
- reservedResource)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("unreserving node with reservation size: "
- + reservedResource
- + " in order to allocate container with size: " + resourceNeedUnreserve);
+ Resource reservedResource = reservedContainer.getReservedResource();
+
+ // make sure we unreserve one with at least the same amount of
+ // resources, otherwise could affect capacity limits
+ if (Resources.fitsIn(rc, clusterResource, resourceNeedUnreserve,
+ reservedResource)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "unreserving node with reservation size: " + reservedResource
+ + " in order to allocate container with size: "
+ + resourceNeedUnreserve);
+ }
+ return nodeId;
}
- return nodeId;
}
}
+ return null;
+ } finally {
+ writeLock.unlock();
}
- return null;
}
- public synchronized void setHeadroomProvider(
+ public void setHeadroomProvider(
CapacityHeadroomProvider headroomProvider) {
- this.headroomProvider = headroomProvider;
- }
-
- public synchronized CapacityHeadroomProvider getHeadroomProvider() {
- return headroomProvider;
+ try {
+ writeLock.lock();
+ this.headroomProvider = headroomProvider;
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
- public synchronized Resource getHeadroom() {
- if (headroomProvider != null) {
- return headroomProvider.getHeadroom();
+ public Resource getHeadroom() {
+ try {
+ readLock.lock();
+ if (headroomProvider != null) {
+ return headroomProvider.getHeadroom();
+ }
+ return super.getHeadroom();
+ } finally {
+ readLock.unlock();
}
- return super.getHeadroom();
+
}
@Override
- public synchronized void transferStateFromPreviousAttempt(
+ public void transferStateFromPreviousAttempt(
SchedulerApplicationAttempt appAttempt) {
- super.transferStateFromPreviousAttempt(appAttempt);
- this.headroomProvider =
- ((FiCaSchedulerApp) appAttempt).getHeadroomProvider();
+ try {
+ writeLock.lock();
+ super.transferStateFromPreviousAttempt(appAttempt);
+ this.headroomProvider = ((FiCaSchedulerApp) appAttempt).headroomProvider;
+ } finally {
+ writeLock.unlock();
+ }
}
public boolean reserveIncreasedContainer(SchedulerRequestKey schedulerKey,
@@ -444,11 +455,11 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
// Update the node
node.reserveResource(this, schedulerKey, rmContainer);
-
+
// Succeeded
return true;
}
-
+
return false;
}
@@ -515,9 +526,12 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
showRequests();
}
- synchronized (this) {
+ try {
+ writeLock.lock();
return containerAllocator.assignContainers(clusterResource, node,
schedulingMode, currentResourceLimits, reservedContainer);
+ } finally {
+ writeLock.unlock();
}
}
@@ -625,23 +639,33 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
* Capacity Scheduler.
*/
@Override
- public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
- ApplicationResourceUsageReport report = super.getResourceUsageReport();
- Resource cluster = rmContext.getScheduler().getClusterResource();
- Resource totalPartitionRes =
- rmContext.getNodeLabelManager()
- .getResourceByLabel(getAppAMNodePartitionName(), cluster);
- ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
- if (!calc.isInvalidDivisor(totalPartitionRes)) {
- float queueAbsMaxCapPerPartition =
- ((AbstractCSQueue)getQueue()).getQueueCapacities()
- .getAbsoluteCapacity(getAppAMNodePartitionName());
- float queueUsagePerc =
- calc.divide(totalPartitionRes, report.getUsedResources(),
- Resources.multiply(totalPartitionRes,
- queueAbsMaxCapPerPartition)) * 100;
- report.setQueueUsagePercentage(queueUsagePerc);
+ public ApplicationResourceUsageReport getResourceUsageReport() {
+ try {
+ // Use write lock here because
+ // SchedulerApplicationAttempt#getResourceUsageReport updated fields
+ // TODO: improve this
+ writeLock.lock();
+ ApplicationResourceUsageReport report = super.getResourceUsageReport();
+ Resource cluster = rmContext.getScheduler().getClusterResource();
+ Resource totalPartitionRes =
+ rmContext.getNodeLabelManager().getResourceByLabel(
+ getAppAMNodePartitionName(), cluster);
+ ResourceCalculator calc =
+ rmContext.getScheduler().getResourceCalculator();
+ if (!calc.isInvalidDivisor(totalPartitionRes)) {
+ float queueAbsMaxCapPerPartition =
+ ((AbstractCSQueue) getQueue()).getQueueCapacities()
+ .getAbsoluteCapacity(getAppAMNodePartitionName());
+ float queueUsagePerc = calc.divide(totalPartitionRes,
+ report.getUsedResources(),
+ Resources.multiply(totalPartitionRes, queueAbsMaxCapPerPartition))
+ * 100;
+ report.setQueueUsagePercentage(queueUsagePerc);
+ }
+ return report;
+ } finally {
+ writeLock.unlock();
}
- return report;
+
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-3140. Improve locks in
AbstractCSQueue/LeafQueue/ParentQueue. Contributed by Wangda Tan
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 3e9785f..ffb6892 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -107,68 +107,77 @@ public class ParentQueue extends AbstractCSQueue {
", fullname=" + getQueuePath());
}
- synchronized void setupQueueConfigs(Resource clusterResource)
+ void setupQueueConfigs(Resource clusterResource)
throws IOException {
- super.setupQueueConfigs(clusterResource);
- StringBuilder aclsString = new StringBuilder();
- for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
- aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
- }
+ try {
+ writeLock.lock();
+ super.setupQueueConfigs(clusterResource);
+ StringBuilder aclsString = new StringBuilder();
+ for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
+ aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
+ }
- StringBuilder labelStrBuilder = new StringBuilder();
- if (accessibleLabels != null) {
- for (String s : accessibleLabels) {
- labelStrBuilder.append(s);
- labelStrBuilder.append(",");
+ StringBuilder labelStrBuilder = new StringBuilder();
+ if (accessibleLabels != null) {
+ for (String s : accessibleLabels) {
+ labelStrBuilder.append(s);
+ labelStrBuilder.append(",");
+ }
}
- }
- LOG.info(queueName +
- ", capacity=" + this.queueCapacities.getCapacity() +
- ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
- ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() +
- ", absoluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() +
- ", state=" + state +
- ", acls=" + aclsString +
- ", labels=" + labelStrBuilder.toString() + "\n" +
- ", reservationsContinueLooking=" + reservationsContinueLooking);
+ LOG.info(queueName + ", capacity=" + this.queueCapacities.getCapacity()
+ + ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity()
+ + ", maxCapacity=" + this.queueCapacities.getMaximumCapacity()
+ + ", absoluteMaxCapacity=" + this.queueCapacities
+ .getAbsoluteMaximumCapacity() + ", state=" + state + ", acls="
+ + aclsString + ", labels=" + labelStrBuilder.toString() + "\n"
+ + ", reservationsContinueLooking=" + reservationsContinueLooking);
+ } finally {
+ writeLock.unlock();
+ }
}
private static float PRECISION = 0.0005f; // 0.05% precision
- synchronized void setChildQueues(Collection<CSQueue> childQueues) {
- // Validate
- float childCapacities = 0;
- for (CSQueue queue : childQueues) {
- childCapacities += queue.getCapacity();
- }
- float delta = Math.abs(1.0f - childCapacities); // crude way to check
- // allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) ||
- ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
- throw new IllegalArgumentException("Illegal" +
- " capacity of " + childCapacities +
- " for children of queue " + queueName);
- }
- // check label capacities
- for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
- float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
- // check children's labels
- float sum = 0;
+
+ void setChildQueues(Collection<CSQueue> childQueues) {
+ try {
+ writeLock.lock();
+ // Validate
+ float childCapacities = 0;
for (CSQueue queue : childQueues) {
- sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+ childCapacities += queue.getCapacity();
}
- if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
- || (capacityByLabel == 0) && (sum > 0)) {
- throw new IllegalArgumentException("Illegal" + " capacity of "
- + sum + " for children of queue " + queueName
- + " for label=" + nodeLabel);
+ float delta = Math.abs(1.0f - childCapacities); // crude way to check
+ // allow capacities being set to 0, and enforce child 0 if parent is 0
+ if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) || (
+ (queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
+ throw new IllegalArgumentException(
+ "Illegal" + " capacity of " + childCapacities
+ + " for children of queue " + queueName);
}
- }
-
- this.childQueues.clear();
- this.childQueues.addAll(childQueues);
- if (LOG.isDebugEnabled()) {
- LOG.debug("setChildQueues: " + getChildQueuesToPrint());
+ // check label capacities
+ for (String nodeLabel : queueCapacities.getExistingNodeLabels()) {
+ float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
+ // check children's labels
+ float sum = 0;
+ for (CSQueue queue : childQueues) {
+ sum += queue.getQueueCapacities().getCapacity(nodeLabel);
+ }
+ if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
+ || (capacityByLabel == 0) && (sum > 0)) {
+ throw new IllegalArgumentException(
+ "Illegal" + " capacity of " + sum + " for children of queue "
+ + queueName + " for label=" + nodeLabel);
+ }
+ }
+
+ this.childQueues.clear();
+ this.childQueues.addAll(childQueues);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("setChildQueues: " + getChildQueuesToPrint());
+ }
+ } finally {
+ writeLock.unlock();
}
}
@@ -179,53 +188,70 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized QueueInfo getQueueInfo(
+ public QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
- QueueInfo queueInfo = getQueueInfo();
-
- List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
- if (includeChildQueues) {
- for (CSQueue child : childQueues) {
- // Get queue information recursively?
- childQueuesInfo.add(
- child.getQueueInfo(recursive, recursive));
+ try {
+ readLock.lock();
+ QueueInfo queueInfo = getQueueInfo();
+
+ List<QueueInfo> childQueuesInfo = new ArrayList<>();
+ if (includeChildQueues) {
+ for (CSQueue child : childQueues) {
+ // Get queue information recursively?
+ childQueuesInfo.add(child.getQueueInfo(recursive, recursive));
+ }
}
+ queueInfo.setChildQueues(childQueuesInfo);
+
+ return queueInfo;
+ } finally {
+ readLock.unlock();
}
- queueInfo.setChildQueues(childQueuesInfo);
-
- return queueInfo;
+
}
- private synchronized QueueUserACLInfo getUserAclInfo(
+ private QueueUserACLInfo getUserAclInfo(
UserGroupInformation user) {
- QueueUserACLInfo userAclInfo =
- recordFactory.newRecordInstance(QueueUserACLInfo.class);
- List<QueueACL> operations = new ArrayList<QueueACL>();
- for (QueueACL operation : QueueACL.values()) {
- if (hasAccess(operation, user)) {
- operations.add(operation);
- }
+ try {
+ readLock.lock();
+ QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
+ QueueUserACLInfo.class);
+ List<QueueACL> operations = new ArrayList<QueueACL>();
+ for (QueueACL operation : QueueACL.values()) {
+ if (hasAccess(operation, user)) {
+ operations.add(operation);
+ }
+ }
+
+ userAclInfo.setQueueName(getQueueName());
+ userAclInfo.setUserAcls(operations);
+ return userAclInfo;
+ } finally {
+ readLock.unlock();
}
- userAclInfo.setQueueName(getQueueName());
- userAclInfo.setUserAcls(operations);
- return userAclInfo;
}
@Override
- public synchronized List<QueueUserACLInfo> getQueueUserAclInfo(
+ public List<QueueUserACLInfo> getQueueUserAclInfo(
UserGroupInformation user) {
- List<QueueUserACLInfo> userAcls = new ArrayList<QueueUserACLInfo>();
-
- // Add parent queue acls
- userAcls.add(getUserAclInfo(user));
-
- // Add children queue acls
- for (CSQueue child : childQueues) {
- userAcls.addAll(child.getQueueUserAclInfo(user));
+ try {
+ readLock.lock();
+ List<QueueUserACLInfo> userAcls = new ArrayList<>();
+
+ // Add parent queue acls
+ userAcls.add(getUserAclInfo(user));
+
+ // Add children queue acls
+ for (CSQueue child : childQueues) {
+ userAcls.addAll(child.getQueueUserAclInfo(user));
+ }
+
+ return userAcls;
+ } finally {
+ readLock.unlock();
}
-
- return userAcls;
+
}
public String toString() {
@@ -240,52 +266,59 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ public void reinitialize(CSQueue newlyParsedQueue,
Resource clusterResource) throws IOException {
- // Sanity check
- if (!(newlyParsedQueue instanceof ParentQueue) ||
- !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
- throw new IOException("Trying to reinitialize " + getQueuePath() +
- " from " + newlyParsedQueue.getQueuePath());
- }
+ try {
+ writeLock.lock();
+ // Sanity check
+ if (!(newlyParsedQueue instanceof ParentQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
+ }
- ParentQueue newlyParsedParentQueue = (ParentQueue)newlyParsedQueue;
+ ParentQueue newlyParsedParentQueue = (ParentQueue) newlyParsedQueue;
- // Set new configs
- setupQueueConfigs(clusterResource);
+ // Set new configs
+ setupQueueConfigs(clusterResource);
- // Re-configure existing child queues and add new ones
- // The CS has already checked to ensure all existing child queues are present!
- Map<String, CSQueue> currentChildQueues = getQueues(childQueues);
- Map<String, CSQueue> newChildQueues =
- getQueues(newlyParsedParentQueue.childQueues);
- for (Map.Entry<String, CSQueue> e : newChildQueues.entrySet()) {
- String newChildQueueName = e.getKey();
- CSQueue newChildQueue = e.getValue();
+ // Re-configure existing child queues and add new ones
+ // The CS has already checked to ensure all existing child queues are present!
+ Map<String, CSQueue> currentChildQueues = getQueues(childQueues);
+ Map<String, CSQueue> newChildQueues = getQueues(
+ newlyParsedParentQueue.childQueues);
+ for (Map.Entry<String, CSQueue> e : newChildQueues.entrySet()) {
+ String newChildQueueName = e.getKey();
+ CSQueue newChildQueue = e.getValue();
- CSQueue childQueue = currentChildQueues.get(newChildQueueName);
-
- // Check if the child-queue already exists
- if (childQueue != null) {
- // Re-init existing child queues
- childQueue.reinitialize(newChildQueue, clusterResource);
- LOG.info(getQueueName() + ": re-configured queue: " + childQueue);
- } else {
- // New child queue, do not re-init
-
- // Set parent to 'this'
- newChildQueue.setParent(this);
-
- // Save in list of current child queues
- currentChildQueues.put(newChildQueueName, newChildQueue);
-
- LOG.info(getQueueName() + ": added new child queue: " + newChildQueue);
+ CSQueue childQueue = currentChildQueues.get(newChildQueueName);
+
+ // Check if the child-queue already exists
+ if (childQueue != null) {
+ // Re-init existing child queues
+ childQueue.reinitialize(newChildQueue, clusterResource);
+ LOG.info(getQueueName() + ": re-configured queue: " + childQueue);
+ } else{
+ // New child queue, do not re-init
+
+ // Set parent to 'this'
+ newChildQueue.setParent(this);
+
+ // Save in list of current child queues
+ currentChildQueues.put(newChildQueueName, newChildQueue);
+
+ LOG.info(
+ getQueueName() + ": added new child queue: " + newChildQueue);
+ }
}
- }
- // Re-sort all queues
- childQueues.clear();
- childQueues.addAll(currentChildQueues.values());
+ // Re-sort all queues
+ childQueues.clear();
+ childQueues.addAll(currentChildQueues.values());
+ } finally {
+ writeLock.unlock();
+ }
}
Map<String, CSQueue> getQueues(Set<CSQueue> queues) {
@@ -299,21 +332,24 @@ public class ParentQueue extends AbstractCSQueue {
@Override
public void submitApplication(ApplicationId applicationId, String user,
String queue) throws AccessControlException {
-
- synchronized (this) {
+
+ try {
+ writeLock.lock();
// Sanity check
if (queue.equals(queueName)) {
- throw new AccessControlException("Cannot submit application " +
- "to non-leaf queue: " + queueName);
+ throw new AccessControlException(
+ "Cannot submit application " + "to non-leaf queue: " + queueName);
}
-
+
if (state != QueueState.RUNNING) {
- throw new AccessControlException("Queue " + getQueuePath() +
- " is STOPPED. Cannot accept submission of application: " +
- applicationId);
+ throw new AccessControlException("Queue " + getQueuePath()
+ + " is STOPPED. Cannot accept submission of application: "
+ + applicationId);
}
addApplication(applicationId, user);
+ } finally {
+ writeLock.unlock();
}
// Inform the parent queue
@@ -342,24 +378,26 @@ public class ParentQueue extends AbstractCSQueue {
// finish attempt logic.
}
- private synchronized void addApplication(ApplicationId applicationId,
+ private void addApplication(ApplicationId applicationId,
String user) {
- ++numApplications;
+ try {
+ writeLock.lock();
+ ++numApplications;
- LOG.info("Application added -" +
- " appId: " + applicationId +
- " user: " + user +
- " leaf-queue of parent: " + getQueueName() +
- " #applications: " + getNumApplications());
+ LOG.info(
+ "Application added -" + " appId: " + applicationId + " user: " + user
+ + " leaf-queue of parent: " + getQueueName() + " #applications: "
+ + getNumApplications());
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
public void finishApplication(ApplicationId application, String user) {
-
- synchronized (this) {
- removeApplication(application, user);
- }
+
+ removeApplication(application, user);
// Inform the parent queue
if (parent != null) {
@@ -367,16 +405,18 @@ public class ParentQueue extends AbstractCSQueue {
}
}
- private synchronized void removeApplication(ApplicationId applicationId,
+ private void removeApplication(ApplicationId applicationId,
String user) {
-
- --numApplications;
-
- LOG.info("Application removed -" +
- " appId: " + applicationId +
- " user: " + user +
- " leaf-queue of parent: " + getQueueName() +
- " #applications: " + getNumApplications());
+ try {
+ writeLock.lock();
+ --numApplications;
+
+ LOG.info("Application removed -" + " appId: " + applicationId + " user: "
+ + user + " leaf-queue of parent: " + getQueueName()
+ + " #applications: " + getNumApplications());
+ } finally {
+ writeLock.unlock();
+ }
}
private String getParentName() {
@@ -384,183 +424,181 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized CSAssignment assignContainers(Resource clusterResource,
+ public CSAssignment assignContainers(Resource clusterResource,
FiCaSchedulerNode node, ResourceLimits resourceLimits,
SchedulingMode schedulingMode) {
- // if our queue cannot access this node, just return
- if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
- && !accessibleToPartition(node.getPartition())) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip this queue=" + getQueuePath()
- + ", because it is not able to access partition=" + node
- .getPartition());
- }
-
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParentName(), getQueueName(), ActivityState.REJECTED,
- ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + node
+ try {
+ writeLock.lock();
+ // if our queue cannot access this node, just return
+ if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
+ && !accessibleToPartition(node.getPartition())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip this queue=" + getQueuePath()
+ + ", because it is not able to access partition=" + node
.getPartition());
- if (rootQueue) {
- ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
- node);
- }
+ }
- return CSAssignment.NULL_ASSIGNMENT;
- }
-
- // Check if this queue need more resource, simply skip allocation if this
- // queue doesn't need more resources.
- if (!super.hasPendingResourceRequest(node.getPartition(),
- clusterResource, schedulingMode)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip this queue=" + getQueuePath()
- + ", because it doesn't need more resource, schedulingMode="
- + schedulingMode.name() + " node-partition=" + node.getPartition());
- }
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParentName(), getQueueName(), ActivityState.REJECTED,
+ ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + node
+ .getPartition());
+ if (rootQueue) {
+ ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
+ node);
+ }
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParentName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
- if (rootQueue) {
- ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
- node);
+ return CSAssignment.NULL_ASSIGNMENT;
}
- return CSAssignment.NULL_ASSIGNMENT;
- }
-
- CSAssignment assignment =
- new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
-
- while (canAssign(clusterResource, node)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Trying to assign containers to child-queue of "
- + getQueueName());
- }
-
- // Are we over maximum-capacity for this queue?
- // This will also consider parent's limits and also continuous reservation
- // looking
- if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
- resourceLimits, Resources.createResource(
- getMetrics().getReservedMB(), getMetrics()
- .getReservedVirtualCores()), schedulingMode)) {
+ // Check if this queue need more resource, simply skip allocation if this
+ // queue doesn't need more resources.
+ if (!super.hasPendingResourceRequest(node.getPartition(), clusterResource,
+ schedulingMode)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip this queue=" + getQueuePath()
+ + ", because it doesn't need more resource, schedulingMode="
+ + schedulingMode.name() + " node-partition=" + node
+ .getPartition());
+ }
ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
getParentName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.QUEUE_MAX_CAPACITY_LIMIT);
+ ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
if (rootQueue) {
ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
node);
}
- break;
+ return CSAssignment.NULL_ASSIGNMENT;
}
- // Schedule
- CSAssignment assignedToChild =
- assignContainersToChildQueues(clusterResource, node, resourceLimits,
- schedulingMode);
- assignment.setType(assignedToChild.getType());
-
- // Done if no child-queue assigned anything
- if (Resources.greaterThan(
- resourceCalculator, clusterResource,
- assignedToChild.getResource(), Resources.none())) {
+ CSAssignment assignment = new CSAssignment(Resources.createResource(0, 0),
+ NodeType.NODE_LOCAL);
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParentName(), getQueueName(), ActivityState.ACCEPTED,
- ActivityDiagnosticConstant.EMPTY);
+ while (canAssign(clusterResource, node)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Trying to assign containers to child-queue of "
+ + getQueueName());
+ }
- if (node.getReservedContainer() == null) {
- if (rootQueue) {
- ActivitiesLogger.NODE.finishAllocatedNodeAllocation(
- activitiesManager, node,
- assignedToChild.getAssignmentInformation()
- .getFirstAllocatedOrReservedContainerId(),
- AllocationState.ALLOCATED);
- }
- } else {
+ // Are we over maximum-capacity for this queue?
+ // This will also consider parent's limits and also continuous reservation
+ // looking
+ if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
+ resourceLimits, Resources
+ .createResource(getMetrics().getReservedMB(),
+ getMetrics().getReservedVirtualCores()), schedulingMode)) {
+
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParentName(), getQueueName(), ActivityState.SKIPPED,
+ ActivityDiagnosticConstant.QUEUE_MAX_CAPACITY_LIMIT);
if (rootQueue) {
- ActivitiesLogger.NODE.finishAllocatedNodeAllocation(
- activitiesManager, node,
- assignedToChild.getAssignmentInformation()
- .getFirstAllocatedOrReservedContainerId(),
- AllocationState.RESERVED);
+ ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
+ node);
}
+
+ break;
}
- // Track resource utilization for the parent-queue
- allocateResource(clusterResource, assignedToChild.getResource(),
- node.getPartition(), assignedToChild.isIncreasedAllocation());
-
- // Track resource utilization in this pass of the scheduler
- Resources
- .addTo(assignment.getResource(), assignedToChild.getResource());
- Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
- assignedToChild.getAssignmentInformation().getAllocated());
- Resources.addTo(assignment.getAssignmentInformation().getReserved(),
- assignedToChild.getAssignmentInformation().getReserved());
- assignment.getAssignmentInformation().incrAllocations(
- assignedToChild.getAssignmentInformation().getNumAllocations());
- assignment.getAssignmentInformation().incrReservations(
- assignedToChild.getAssignmentInformation().getNumReservations());
- assignment
- .getAssignmentInformation()
- .getAllocationDetails()
- .addAll(
- assignedToChild.getAssignmentInformation().getAllocationDetails());
- assignment
- .getAssignmentInformation()
- .getReservationDetails()
- .addAll(
+ // Schedule
+ CSAssignment assignedToChild = assignContainersToChildQueues(
+ clusterResource, node, resourceLimits, schedulingMode);
+ assignment.setType(assignedToChild.getType());
+
+ // Done if no child-queue assigned anything
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ assignedToChild.getResource(), Resources.none())) {
+
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParentName(), getQueueName(), ActivityState.ACCEPTED,
+ ActivityDiagnosticConstant.EMPTY);
+
+ if (node.getReservedContainer() == null) {
+ if (rootQueue) {
+ ActivitiesLogger.NODE.finishAllocatedNodeAllocation(
+ activitiesManager, node,
+ assignedToChild.getAssignmentInformation()
+ .getFirstAllocatedOrReservedContainerId(),
+ AllocationState.ALLOCATED);
+ }
+ } else{
+ if (rootQueue) {
+ ActivitiesLogger.NODE.finishAllocatedNodeAllocation(
+ activitiesManager, node,
+ assignedToChild.getAssignmentInformation()
+ .getFirstAllocatedOrReservedContainerId(),
+ AllocationState.RESERVED);
+ }
+ }
+
+ // Track resource utilization for the parent-queue
+ allocateResource(clusterResource, assignedToChild.getResource(),
+ node.getPartition(), assignedToChild.isIncreasedAllocation());
+
+ // Track resource utilization in this pass of the scheduler
+ Resources.addTo(assignment.getResource(),
+ assignedToChild.getResource());
+ Resources.addTo(assignment.getAssignmentInformation().getAllocated(),
+ assignedToChild.getAssignmentInformation().getAllocated());
+ Resources.addTo(assignment.getAssignmentInformation().getReserved(),
+ assignedToChild.getAssignmentInformation().getReserved());
+ assignment.getAssignmentInformation().incrAllocations(
+ assignedToChild.getAssignmentInformation().getNumAllocations());
+ assignment.getAssignmentInformation().incrReservations(
+ assignedToChild.getAssignmentInformation().getNumReservations());
+ assignment.getAssignmentInformation().getAllocationDetails().addAll(
+ assignedToChild.getAssignmentInformation()
+ .getAllocationDetails());
+ assignment.getAssignmentInformation().getReservationDetails().addAll(
assignedToChild.getAssignmentInformation()
.getReservationDetails());
- assignment.setIncreasedAllocation(assignedToChild
- .isIncreasedAllocation());
-
- LOG.info("assignedContainer" +
- " queue=" + getQueueName() +
- " usedCapacity=" + getUsedCapacity() +
- " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + queueUsage.getUsed() +
- " cluster=" + clusterResource);
-
- } else {
- assignment.setSkippedType(assignedToChild.getSkippedType());
+ assignment.setIncreasedAllocation(
+ assignedToChild.isIncreasedAllocation());
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParentName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.EMPTY);
- if (rootQueue) {
- ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
- node);
- }
+ LOG.info("assignedContainer" + " queue=" + getQueueName()
+ + " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed()
+ + " cluster=" + clusterResource);
- break;
- }
+ } else{
+ assignment.setSkippedType(assignedToChild.getSkippedType());
- if (LOG.isDebugEnabled()) {
- LOG.debug("ParentQ=" + getQueueName()
- + " assignedSoFarInThisIteration=" + assignment.getResource()
- + " usedCapacity=" + getUsedCapacity()
- + " absoluteUsedCapacity=" + getAbsoluteUsedCapacity());
- }
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParentName(), getQueueName(), ActivityState.SKIPPED,
+ ActivityDiagnosticConstant.EMPTY);
+ if (rootQueue) {
+ ActivitiesLogger.NODE.finishSkippedNodeAllocation(activitiesManager,
+ node);
+ }
+
+ break;
+ }
- // Do not assign more than one container if this isn't the root queue
- // or if we've already assigned an off-switch container
- if (!rootQueue || assignment.getType() == NodeType.OFF_SWITCH) {
if (LOG.isDebugEnabled()) {
- if (rootQueue && assignment.getType() == NodeType.OFF_SWITCH) {
- LOG.debug("Not assigning more than one off-switch container," +
- " assignments so far: " + assignment);
+ LOG.debug(
+ "ParentQ=" + getQueueName() + " assignedSoFarInThisIteration="
+ + assignment.getResource() + " usedCapacity="
+ + getUsedCapacity() + " absoluteUsedCapacity="
+ + getAbsoluteUsedCapacity());
+ }
+
+ // Do not assign more than one container if this isn't the root queue
+ // or if we've already assigned an off-switch container
+ if (!rootQueue || assignment.getType() == NodeType.OFF_SWITCH) {
+ if (LOG.isDebugEnabled()) {
+ if (rootQueue && assignment.getType() == NodeType.OFF_SWITCH) {
+ LOG.debug("Not assigning more than one off-switch container,"
+ + " assignments so far: " + assignment);
+ }
}
+ break;
}
- break;
}
- }
-
- return assignment;
+
+ return assignment;
+ } finally {
+ writeLock.unlock();
+ }
}
private boolean canAssign(Resource clusterResource, FiCaSchedulerNode node) {
@@ -628,7 +666,7 @@ public class ParentQueue extends AbstractCSQueue {
return childrenList.iterator();
}
- private synchronized CSAssignment assignContainersToChildQueues(
+ private CSAssignment assignContainersToChildQueues(
Resource cluster, FiCaSchedulerNode node, ResourceLimits limits,
SchedulingMode schedulingMode) {
CSAssignment assignment = CSAssignment.NULL_ASSIGNMENT;
@@ -717,39 +755,45 @@ public class ParentQueue extends AbstractCSQueue {
}
}
- private synchronized void internalReleaseResource(Resource clusterResource,
+ private void internalReleaseResource(Resource clusterResource,
FiCaSchedulerNode node, Resource releasedResource, boolean changeResource,
CSQueue completedChildQueue, boolean sortQueues) {
- super.releaseResource(clusterResource,
- releasedResource, node.getPartition(),
- changeResource);
+ try {
+ writeLock.lock();
+ super.releaseResource(clusterResource, releasedResource,
+ node.getPartition(), changeResource);
- if (LOG.isDebugEnabled()) {
- LOG.debug("completedContainer " + this + ", cluster=" + clusterResource);
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "completedContainer " + this + ", cluster=" + clusterResource);
+ }
- // Note that this is using an iterator on the childQueues so this can't
- // be called if already within an iterator for the childQueues. Like
- // from assignContainersToChildQueues.
- if (sortQueues) {
- // reinsert the updated queue
- for (Iterator<CSQueue> iter = childQueues.iterator(); iter.hasNext();) {
- CSQueue csqueue = iter.next();
- if (csqueue.equals(completedChildQueue)) {
- iter.remove();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Re-sorting completed queue: " + csqueue);
+ // Note that this is using an iterator on the childQueues so this can't
+ // be called if already within an iterator for the childQueues. Like
+ // from assignContainersToChildQueues.
+ if (sortQueues) {
+ // reinsert the updated queue
+ for (Iterator<CSQueue> iter = childQueues.iterator();
+ iter.hasNext(); ) {
+ CSQueue csqueue = iter.next();
+ if (csqueue.equals(completedChildQueue)) {
+ iter.remove();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Re-sorting completed queue: " + csqueue);
+ }
+ childQueues.add(csqueue);
+ break;
}
- childQueues.add(csqueue);
- break;
}
}
- }
- // If we skipped sort queue this time, we need to resort queues to make
- // sure we allocate from least usage (or order defined by queue policy)
- // queues.
- needToResortQueuesAtNextAllocation = !sortQueues;
+ // If we skipped sort queue this time, we need to resort queues to make
+ // sure we allocate from least usage (or order defined by queue policy)
+ // queues.
+ needToResortQueuesAtNextAllocation = !sortQueues;
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
@@ -806,24 +850,35 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized void updateClusterResource(Resource clusterResource,
+ public void updateClusterResource(Resource clusterResource,
ResourceLimits resourceLimits) {
- // Update all children
- for (CSQueue childQueue : childQueues) {
- // Get ResourceLimits of child queue before assign containers
- ResourceLimits childLimits = getResourceLimitsOfChild(childQueue,
- clusterResource, resourceLimits.getLimit(),
- RMNodeLabelsManager.NO_LABEL);
- childQueue.updateClusterResource(clusterResource, childLimits);
+ try {
+ writeLock.lock();
+ // Update all children
+ for (CSQueue childQueue : childQueues) {
+ // Get ResourceLimits of child queue before assign containers
+ ResourceLimits childLimits = getResourceLimitsOfChild(childQueue,
+ clusterResource, resourceLimits.getLimit(),
+ RMNodeLabelsManager.NO_LABEL);
+ childQueue.updateClusterResource(clusterResource, childLimits);
+ }
+
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ minimumAllocation, this, labelManager, null);
+ } finally {
+ writeLock.unlock();
}
-
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
}
@Override
- public synchronized List<CSQueue> getChildQueues() {
- return new ArrayList<CSQueue>(childQueues);
+ public List<CSQueue> getChildQueues() {
+ try {
+ readLock.lock();
+ return new ArrayList<CSQueue>(childQueues);
+ } finally {
+ readLock.unlock();
+ }
+
}
@Override
@@ -832,13 +887,18 @@ public class ParentQueue extends AbstractCSQueue {
if (rmContainer.getState().equals(RMContainerState.COMPLETED)) {
return;
}
- // Careful! Locking order is important!
- synchronized (this) {
- FiCaSchedulerNode node =
- scheduler.getNode(rmContainer.getContainer().getNodeId());
+
+ // Careful! Locking order is important!
+ try {
+ writeLock.lock();
+ FiCaSchedulerNode node = scheduler.getNode(
+ rmContainer.getContainer().getNodeId());
allocateResource(clusterResource,
rmContainer.getContainer().getResource(), node.getPartition(), false);
+ } finally {
+ writeLock.unlock();
}
+
if (parent != null) {
parent.recoverContainer(clusterResource, attempt, rmContainer);
}
@@ -851,11 +911,17 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized void collectSchedulerApplications(
+ public void collectSchedulerApplications(
Collection<ApplicationAttemptId> apps) {
- for (CSQueue queue : childQueues) {
- queue.collectSchedulerApplications(apps);
+ try {
+ readLock.lock();
+ for (CSQueue queue : childQueues) {
+ queue.collectSchedulerApplications(apps);
+ }
+ } finally {
+ readLock.unlock();
}
+
}
@Override
@@ -897,44 +963,49 @@ public class ParentQueue extends AbstractCSQueue {
}
}
- public synchronized int getNumApplications() {
+ public int getNumApplications() {
return numApplications;
}
- synchronized void allocateResource(Resource clusterResource,
+ void allocateResource(Resource clusterResource,
Resource resource, String nodePartition, boolean changeContainerResource) {
- super.allocateResource(clusterResource, resource, nodePartition,
- changeContainerResource);
-
- /**
- * check if we need to kill (killable) containers if maximum resource violated.
- * Doing this because we will deduct killable resource when going from root.
- * For example:
- * <pre>
- * Root
- * / \
- * a b
- * / \
- * a1 a2
- * </pre>
- *
- * a: max=10G, used=10G, killable=2G
- * a1: used=8G, killable=2G
- * a2: used=2G, pending=2G, killable=0G
- *
- * When we get queue-a to allocate resource, even if queue-a
- * reaches its max resource, we deduct its used by killable, so we can allocate
- * at most 2G resources. ResourceLimits passed down to a2 has headroom set to 2G.
- *
- * If scheduler finds a 2G available resource in existing cluster, and assigns it
- * to a2, now a2's used= 2G + 2G = 4G, and a's used = 8G + 4G = 12G > 10G
- *
- * When this happens, we have to preempt killable container (on same or different
- * nodes) of parent queue to avoid violating parent's max resource.
- */
- if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
- < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
- killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+ try {
+ writeLock.lock();
+ super.allocateResource(clusterResource, resource, nodePartition,
+ changeContainerResource);
+
+ /**
+ * check if we need to kill (killable) containers if maximum resource violated.
+ * Doing this because we will deduct killable resource when going from root.
+ * For example:
+ * <pre>
+ * Root
+ * / \
+ * a b
+ * / \
+ * a1 a2
+ * </pre>
+ *
+ * a: max=10G, used=10G, killable=2G
+ * a1: used=8G, killable=2G
+ * a2: used=2G, pending=2G, killable=0G
+ *
+ * When we get queue-a to allocate resource, even if queue-a
+ * reaches its max resource, we deduct its used by killable, so we can allocate
+ * at most 2G resources. ResourceLimits passed down to a2 has headroom set to 2G.
+ *
+ * If scheduler finds a 2G available resource in existing cluster, and assigns it
+ * to a2, now a2's used= 2G + 2G = 4G, and a's used = 8G + 4G = 12G > 10G
+ *
+ * When this happens, we have to preempt killable container (on same or different
+ * nodes) of parent queue to avoid violating parent's max resource.
+ */
+ if (getQueueCapacities().getAbsoluteMaximumCapacity(nodePartition)
+ < getQueueCapacities().getAbsoluteUsedCapacity(nodePartition)) {
+ killContainersToEnforceMaxQueueCapacity(nodePartition, clusterResource);
+ }
+ } finally {
+ writeLock.unlock();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 7b53ad5..a391f25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -79,76 +79,98 @@ public class PlanQueue extends ParentQueue {
}
@Override
- public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ public void reinitialize(CSQueue newlyParsedQueue,
Resource clusterResource) throws IOException {
- // Sanity check
- if (!(newlyParsedQueue instanceof PlanQueue)
- || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
- throw new IOException("Trying to reinitialize " + getQueuePath()
- + " from " + newlyParsedQueue.getQueuePath());
- }
+ try {
+ writeLock.lock();
+ // Sanity check
+ if (!(newlyParsedQueue instanceof PlanQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
+ }
- PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
+ PlanQueue newlyParsedParentQueue = (PlanQueue) newlyParsedQueue;
- if (newlyParsedParentQueue.getChildQueues().size() > 0) {
- throw new IOException(
- "Reservable Queue should not have sub-queues in the"
- + "configuration");
- }
+ if (newlyParsedParentQueue.getChildQueues().size() > 0) {
+ throw new IOException(
+ "Reservable Queue should not have sub-queues in the"
+ + "configuration");
+ }
- // Set new configs
- setupQueueConfigs(clusterResource);
+ // Set new configs
+ setupQueueConfigs(clusterResource);
- updateQuotas(newlyParsedParentQueue.userLimit,
- newlyParsedParentQueue.userLimitFactor,
- newlyParsedParentQueue.maxAppsForReservation,
- newlyParsedParentQueue.maxAppsPerUserForReservation);
+ updateQuotas(newlyParsedParentQueue.userLimit,
+ newlyParsedParentQueue.userLimitFactor,
+ newlyParsedParentQueue.maxAppsForReservation,
+ newlyParsedParentQueue.maxAppsPerUserForReservation);
- // run reinitialize on each existing queue, to trigger absolute cap
- // recomputations
- for (CSQueue res : this.getChildQueues()) {
- res.reinitialize(res, clusterResource);
+ // run reinitialize on each existing queue, to trigger absolute cap
+ // recomputations
+ for (CSQueue res : this.getChildQueues()) {
+ res.reinitialize(res, clusterResource);
+ }
+ showReservationsAsQueues =
+ newlyParsedParentQueue.showReservationsAsQueues;
+ } finally {
+ writeLock.unlock();
}
- showReservationsAsQueues = newlyParsedParentQueue.showReservationsAsQueues;
}
- synchronized void addChildQueue(CSQueue newQueue)
+ void addChildQueue(CSQueue newQueue)
throws SchedulerDynamicEditException {
- if (newQueue.getCapacity() > 0) {
- throw new SchedulerDynamicEditException("Queue " + newQueue
- + " being added has non zero capacity.");
- }
- boolean added = this.childQueues.add(newQueue);
- if (LOG.isDebugEnabled()) {
- LOG.debug("updateChildQueues (action: add queue): " + added + " "
- + getChildQueuesToPrint());
+ try {
+ writeLock.lock();
+ if (newQueue.getCapacity() > 0) {
+ throw new SchedulerDynamicEditException(
+ "Queue " + newQueue + " being added has non zero capacity.");
+ }
+ boolean added = this.childQueues.add(newQueue);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("updateChildQueues (action: add queue): " + added + " "
+ + getChildQueuesToPrint());
+ }
+ } finally {
+ writeLock.unlock();
}
}
- synchronized void removeChildQueue(CSQueue remQueue)
+ void removeChildQueue(CSQueue remQueue)
throws SchedulerDynamicEditException {
- if (remQueue.getCapacity() > 0) {
- throw new SchedulerDynamicEditException("Queue " + remQueue
- + " being removed has non zero capacity.");
- }
- Iterator<CSQueue> qiter = childQueues.iterator();
- while (qiter.hasNext()) {
- CSQueue cs = qiter.next();
- if (cs.equals(remQueue)) {
- qiter.remove();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Removed child queue: {}", cs.getQueueName());
+ try {
+ writeLock.lock();
+ if (remQueue.getCapacity() > 0) {
+ throw new SchedulerDynamicEditException(
+ "Queue " + remQueue + " being removed has non zero capacity.");
+ }
+ Iterator<CSQueue> qiter = childQueues.iterator();
+ while (qiter.hasNext()) {
+ CSQueue cs = qiter.next();
+ if (cs.equals(remQueue)) {
+ qiter.remove();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Removed child queue: {}", cs.getQueueName());
+ }
}
}
+ } finally {
+ writeLock.unlock();
}
}
- protected synchronized float sumOfChildCapacities() {
- float ret = 0;
- for (CSQueue l : childQueues) {
- ret += l.getCapacity();
+ protected float sumOfChildCapacities() {
+ try {
+ writeLock.lock();
+ float ret = 0;
+ for (CSQueue l : childQueues) {
+ ret += l.getCapacity();
+ }
+ return ret;
+ } finally {
+ writeLock.unlock();
}
- return ret;
}
private void updateQuotas(int userLimit, float userLimitFactor,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index 976cf8c..faeb37e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -51,22 +51,28 @@ public class ReservationQueue extends LeafQueue {
}
@Override
- public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ public void reinitialize(CSQueue newlyParsedQueue,
Resource clusterResource) throws IOException {
- // Sanity check
- if (!(newlyParsedQueue instanceof ReservationQueue)
- || !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
- throw new IOException("Trying to reinitialize " + getQueuePath()
- + " from " + newlyParsedQueue.getQueuePath());
- }
- super.reinitialize(newlyParsedQueue, clusterResource);
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
+ try {
+ writeLock.lock();
+ // Sanity check
+ if (!(newlyParsedQueue instanceof ReservationQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
+ }
+ super.reinitialize(newlyParsedQueue, clusterResource);
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ minimumAllocation, this, labelManager, null);
- updateQuotas(parent.getUserLimitForReservation(),
- parent.getUserLimitFactor(),
- parent.getMaxApplicationsForReservations(),
- parent.getMaxApplicationsPerUserForReservation());
+ updateQuotas(parent.getUserLimitForReservation(),
+ parent.getUserLimitFactor(),
+ parent.getMaxApplicationsForReservations(),
+ parent.getMaxApplicationsPerUserForReservation());
+ } finally {
+ writeLock.unlock();
+ }
}
/**
@@ -77,21 +83,26 @@ public class ReservationQueue extends LeafQueue {
* maxCapacity, etc..)
* @throws SchedulerDynamicEditException
*/
- public synchronized void setEntitlement(QueueEntitlement entitlement)
+ public void setEntitlement(QueueEntitlement entitlement)
throws SchedulerDynamicEditException {
- float capacity = entitlement.getCapacity();
- if (capacity < 0 || capacity > 1.0f) {
- throw new SchedulerDynamicEditException(
- "Capacity demand is not in the [0,1] range: " + capacity);
- }
- setCapacity(capacity);
- setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
- // note: we currently set maxCapacity to capacity
- // this might be revised later
- setMaxCapacity(entitlement.getMaxCapacity());
- if (LOG.isDebugEnabled()) {
- LOG.debug("successfully changed to " + capacity + " for queue "
- + this.getQueueName());
+ try {
+ writeLock.lock();
+ float capacity = entitlement.getCapacity();
+ if (capacity < 0 || capacity > 1.0f) {
+ throw new SchedulerDynamicEditException(
+ "Capacity demand is not in the [0,1] range: " + capacity);
+ }
+ setCapacity(capacity);
+ setAbsoluteCapacity(getParent().getAbsoluteCapacity() * getCapacity());
+ // note: we currently set maxCapacity to capacity
+ // this might be revised later
+ setMaxCapacity(entitlement.getMaxCapacity());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("successfully changed to " + capacity + " for queue " + this
+ .getQueueName());
+ }
+ } finally {
+ writeLock.unlock();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index 6fba22a..2614630 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -828,8 +828,8 @@ public class TestContainerResizing {
app.getAppAttemptResourceUsage().getPending().getMemorySize());
// Queue/user/application's usage will be updated
checkUsedResource(rm1, "default", 0 * GB, null);
- Assert.assertEquals(0 * GB, ((LeafQueue) cs.getQueue("default"))
- .getUser("user").getUsed().getMemorySize());
+ // User will be removed
+ Assert.assertNull(((LeafQueue) cs.getQueue("default")).getUser("user"));
Assert.assertEquals(0 * GB,
app.getAppAttemptResourceUsage().getReserved().getMemorySize());
Assert.assertEquals(0 * GB,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-5656. Fix
ReservationACLsTestBase. (Sean Po via asuresh)
Posted by ji...@apache.org.
YARN-5656. Fix ReservationACLsTestBase. (Sean Po via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9f03b403
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9f03b403
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9f03b403
Branch: refs/heads/MAPREDUCE-6608
Commit: 9f03b403ec69658fc57bc0f6b832da0e3c746497
Parents: e45307c
Author: Arun Suresh <as...@apache.org>
Authored: Tue Sep 20 12:27:17 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Tue Sep 20 12:27:17 2016 -0700
----------------------------------------------------------------------
.../reservation/NoOverCommitPolicy.java | 12 -----
.../exceptions/MismatchedUserException.java | 46 --------------------
.../ReservationACLsTestBase.java | 2 +
.../reservation/TestNoOverCommitPolicy.java | 21 ---------
4 files changed, 2 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03b403/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
index 814d4b5..55f1d00 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/NoOverCommitPolicy.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.reservation;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
@@ -39,17 +38,6 @@ public class NoOverCommitPolicy implements SharingPolicy {
public void validate(Plan plan, ReservationAllocation reservation)
throws PlanningException {
- ReservationAllocation oldReservation =
- plan.getReservationById(reservation.getReservationId());
-
- // check updates are using same name
- if (oldReservation != null
- && !oldReservation.getUser().equals(reservation.getUser())) {
- throw new MismatchedUserException(
- "Updating an existing reservation with mismatching user:"
- + oldReservation.getUser() + " != " + reservation.getUser());
- }
-
RLESparseResourceAllocation available = plan.getAvailableResourceOverTime(
reservation.getUser(), reservation.getReservationId(),
reservation.getStartTime(), reservation.getEndTime());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03b403/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java
deleted file mode 100644
index 7b4419b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/exceptions/MismatchedUserException.java
+++ /dev/null
@@ -1,46 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * Exception thrown when an update to an existing reservation is performed
- * by a user that is not the reservation owner.
- */
-@Public
-@Unstable
-public class MismatchedUserException extends PlanningException {
-
- private static final long serialVersionUID = 8313222590561668413L;
-
- public MismatchedUserException(String message) {
- super(message);
- }
-
- public MismatchedUserException(Throwable cause) {
- super(cause);
- }
-
- public MismatchedUserException(String message, Throwable cause) {
- super(message, cause);
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03b403/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
index aa5acc6..c536d8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
@@ -566,6 +566,8 @@ public class ReservationACLsTestBase extends ACLsTestBase {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
out.println("<?xml version=\"1.0\"?>");
out.println("<allocations>");
+ out.println(" <defaultQueueSchedulingPolicy>drf" +
+ "</defaultQueueSchedulingPolicy>");
out.println(" <queue name=\"queueA\">");
out.println(" <aclSubmitReservations>" +
"queueA_user,common_user " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9f03b403/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
index 28dd62e..c5edaf0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestNoOverCommitPolicy.java
@@ -23,10 +23,8 @@ import static org.mockito.Mockito.mock;
import java.io.IOException;
import org.apache.hadoop.yarn.api.records.ReservationDefinition;
-import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.MismatchedUserException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.ResourceOverCommitException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent;
@@ -127,25 +125,6 @@ public class TestNoOverCommitPolicy {
.generateAllocation(initTime, step, f), res, minAlloc), false);
}
- @Test(expected = MismatchedUserException.class)
- public void testUserMismatch() throws IOException, PlanningException {
- // generate allocation from single tenant that exceed capacity
- int[] f = generateData(3600, (int) (0.5 * totCont));
- ReservationDefinition rDef =
- ReservationSystemTestUtil.createSimpleReservationDefinition(
- initTime, initTime + f.length + 1, f.length);
- ReservationId rid = ReservationSystemTestUtil.getNewReservationId();
-
- plan.addReservation(new InMemoryReservationAllocation(rid, rDef, "u1",
- "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil
- .generateAllocation(initTime, step, f), res, minAlloc), false);
-
- // trying to update a reservation with a mismatching user
- plan.updateReservation(new InMemoryReservationAllocation(rid, rDef, "u2",
- "dedicated", initTime, initTime + f.length, ReservationSystemTestUtil
- .generateAllocation(initTime, step, f), res, minAlloc));
- }
-
@Test
public void testMultiTenantPass() throws IOException, PlanningException {
// generate allocation from multiple tenants that barely fit in tot capacity
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: HADOOP-13584. hdoop-aliyun: merge
HADOOP-12756 branch back.
Posted by ji...@apache.org.
HADOOP-13584. hdoop-aliyun: merge HADOOP-12756 branch back.
HADOOP-12756 branch: Incorporate Aliyun OSS file system implementation. Contributors:
Mingfei Shi (mingfei.shi@intel.com)
Genmao Yu (genmao.ygm@alibaba-inc.com)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5707f88d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5707f88d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5707f88d
Branch: refs/heads/MAPREDUCE-6608
Commit: 5707f88d8550346f167e45c2f8c4161eb3957e3a
Parents: e4e72db
Author: Kai Zheng <ka...@intel.com>
Authored: Mon Sep 26 20:42:22 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Mon Sep 26 20:42:22 2016 +0800
----------------------------------------------------------------------
.gitignore | 2 +
hadoop-project/pom.xml | 22 +
.../dev-support/findbugs-exclude.xml | 18 +
hadoop-tools/hadoop-aliyun/pom.xml | 154 ++++++
.../aliyun/oss/AliyunCredentialsProvider.java | 87 +++
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 543 +++++++++++++++++++
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 516 ++++++++++++++++++
.../fs/aliyun/oss/AliyunOSSInputStream.java | 260 +++++++++
.../fs/aliyun/oss/AliyunOSSOutputStream.java | 111 ++++
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 167 ++++++
.../apache/hadoop/fs/aliyun/oss/Constants.java | 113 ++++
.../hadoop/fs/aliyun/oss/package-info.java | 22 +
.../site/markdown/tools/hadoop-aliyun/index.md | 294 ++++++++++
.../fs/aliyun/oss/AliyunOSSTestUtils.java | 77 +++
.../fs/aliyun/oss/TestAliyunCredentials.java | 78 +++
.../oss/TestAliyunOSSFileSystemContract.java | 239 ++++++++
.../oss/TestAliyunOSSFileSystemStore.java | 125 +++++
.../fs/aliyun/oss/TestAliyunOSSInputStream.java | 145 +++++
.../aliyun/oss/TestAliyunOSSOutputStream.java | 91 ++++
.../aliyun/oss/contract/AliyunOSSContract.java | 49 ++
.../contract/TestAliyunOSSContractCreate.java | 35 ++
.../contract/TestAliyunOSSContractDelete.java | 34 ++
.../contract/TestAliyunOSSContractDistCp.java | 44 ++
.../TestAliyunOSSContractGetFileStatus.java | 35 ++
.../contract/TestAliyunOSSContractMkdir.java | 34 ++
.../oss/contract/TestAliyunOSSContractOpen.java | 34 ++
.../contract/TestAliyunOSSContractRename.java | 35 ++
.../contract/TestAliyunOSSContractRootDir.java | 69 +++
.../oss/contract/TestAliyunOSSContractSeek.java | 34 ++
.../src/test/resources/contract/aliyun-oss.xml | 115 ++++
.../src/test/resources/core-site.xml | 46 ++
.../src/test/resources/log4j.properties | 23 +
hadoop-tools/hadoop-tools-dist/pom.xml | 6 +
hadoop-tools/pom.xml | 1 +
34 files changed, 3658 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
index a5d69d0..194862b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,5 @@ hadoop-tools/hadoop-aws/src/test/resources/auth-keys.xml
hadoop-tools/hadoop-aws/src/test/resources/contract-test-options.xml
hadoop-tools/hadoop-azure/src/test/resources/azure-auth-keys.xml
patchprocess/
+hadoop-tools/hadoop-aliyun/src/test/resources/auth-keys.xml
+hadoop-tools/hadoop-aliyun/src/test/resources/contract-test-options.xml
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index d9a01a0..49ea40f 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -439,6 +439,12 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
<version>${project.version}</version>
<classifier>classes</classifier>
@@ -1005,6 +1011,22 @@
<version>4.2.0</version>
</dependency>
+ <dependency>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <version>2.2.1</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-beanutils</groupId>
+ <artifactId>commons-beanutils</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<dependency>
<groupId>xerces</groupId>
<artifactId>xercesImpl</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml
new file mode 100644
index 0000000..40d78d0
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/dev-support/findbugs-exclude.xml
@@ -0,0 +1,18 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FindBugsFilter>
+</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
new file mode 100644
index 0000000..358b18b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-project</artifactId>
+ <version>3.0.0-alpha2-SNAPSHOT</version>
+ <relativePath>../../hadoop-project</relativePath>
+ </parent>
+ <artifactId>hadoop-aliyun</artifactId>
+ <name>Apache Hadoop Aliyun OSS support</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <file.encoding>UTF-8</file.encoding>
+ <downloadSources>true</downloadSources>
+ </properties>
+
+ <profiles>
+ <profile>
+ <id>tests-off</id>
+ <activation>
+ <file>
+ <missing>src/test/resources/auth-keys.xml</missing>
+ </file>
+ </activation>
+ <properties>
+ <maven.test.skip>true</maven.test.skip>
+ </properties>
+ </profile>
+ <profile>
+ <id>tests-on</id>
+ <activation>
+ <file>
+ <exists>src/test/resources/auth-keys.xml</exists>
+ </file>
+ </activation>
+ <properties>
+ <maven.test.skip>false</maven.test.skip>
+ </properties>
+ </profile>
+ </profiles>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>findbugs-maven-plugin</artifactId>
+ <configuration>
+ <findbugsXmlOutput>true</findbugsXmlOutput>
+ <xmlOutput>true</xmlOutput>
+ <excludeFilterFile>${basedir}/dev-support/findbugs-exclude.xml
+ </excludeFilterFile>
+ <effort>Max</effort>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <configuration>
+ <dependencyDetailsEnabled>false</dependencyDetailsEnabled>
+ <dependencyLocationsEnabled>false</dependencyLocationsEnabled>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <configuration>
+ <forkedProcessTimeoutInSeconds>3600</forkedProcessTimeoutInSeconds>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>deplist</id>
+ <phase>compile</phase>
+ <goals>
+ <goal>list</goal>
+ </goals>
+ <configuration>
+ <!-- build a shellprofile -->
+ <outputFile>${project.basedir}/target/hadoop-tools-deps/${project.artifactId}.tools-optional.txt</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>com.aliyun.oss</groupId>
+ <artifactId>aliyun-sdk-oss</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>compile</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-distcp</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-tests</artifactId>
+ <scope>test</scope>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-mapreduce-client-jobclient</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
new file mode 100644
index 0000000..b46c67a
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.common.auth.Credentials;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.auth.DefaultCredentials;
+import com.aliyun.oss.common.auth.InvalidCredentialsException;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Support session credentials for authenticating with Aliyun.
+ */
+public class AliyunCredentialsProvider implements CredentialsProvider {
+ private Credentials credentials = null;
+
+ public AliyunCredentialsProvider(Configuration conf)
+ throws IOException {
+ String accessKeyId;
+ String accessKeySecret;
+ String securityToken;
+ try {
+ accessKeyId = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_ID);
+ accessKeySecret = AliyunOSSUtils.getValueWithKey(conf, ACCESS_KEY_SECRET);
+ } catch (IOException e) {
+ throw new InvalidCredentialsException(e);
+ }
+
+ try {
+ securityToken = AliyunOSSUtils.getValueWithKey(conf, SECURITY_TOKEN);
+ } catch (IOException e) {
+ securityToken = null;
+ }
+
+ if (StringUtils.isEmpty(accessKeyId)
+ || StringUtils.isEmpty(accessKeySecret)) {
+ throw new InvalidCredentialsException(
+ "AccessKeyId and AccessKeySecret should not be null or empty.");
+ }
+
+ if (StringUtils.isNotEmpty(securityToken)) {
+ credentials = new DefaultCredentials(accessKeyId, accessKeySecret,
+ securityToken);
+ } else {
+ credentials = new DefaultCredentials(accessKeyId, accessKeySecret);
+ }
+ }
+
+ @Override
+ public void setCredentials(Credentials creds) {
+ if (creds == null) {
+ throw new InvalidCredentialsException("Credentials should not be null.");
+ }
+
+ credentials = creds;
+ }
+
+ @Override
+ public Credentials getCredentials() {
+ if (credentials == null) {
+ throw new InvalidCredentialsException("Invalid credentials");
+ }
+
+ return credentials;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
new file mode 100644
index 0000000..81e038d
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -0,0 +1,543 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Progressable;
+
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.ObjectListing;
+import com.aliyun.oss.model.ObjectMetadata;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Implementation of {@link FileSystem} for <a href="https://oss.aliyun.com">
+ * Aliyun OSS</a>, used to access OSS blob system in a filesystem style.
+ */
+public class AliyunOSSFileSystem extends FileSystem {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSFileSystem.class);
+ private URI uri;
+ private Path workingDir;
+ private AliyunOSSFileSystemStore store;
+ private int maxKeys;
+
+ @Override
+ public FSDataOutputStream append(Path path, int bufferSize,
+ Progressable progress) throws IOException {
+ throw new IOException("Append is not supported!");
+ }
+
+ @Override
+ public void close() throws IOException {
+ try {
+ store.close();
+ } finally {
+ super.close();
+ }
+ }
+
+ @Override
+ public FSDataOutputStream create(Path path, FsPermission permission,
+ boolean overwrite, int bufferSize, short replication, long blockSize,
+ Progressable progress) throws IOException {
+ String key = pathToKey(path);
+ FileStatus status = null;
+
+ try {
+ // get the status or throw a FNFE
+ status = getFileStatus(path);
+
+ // if the thread reaches here, there is something at the path
+ if (status.isDirectory()) {
+ // path references a directory
+ throw new FileAlreadyExistsException(path + " is a directory");
+ }
+ if (!overwrite) {
+ // path references a file and overwrite is disabled
+ throw new FileAlreadyExistsException(path + " already exists");
+ }
+ LOG.debug("Overwriting file {}", path);
+ } catch (FileNotFoundException e) {
+ // this means the file is not found
+ }
+
+ return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(),
+ store, key, progress, statistics), (Statistics)(null));
+ }
+
+ @Override
+ public boolean delete(Path path, boolean recursive) throws IOException {
+ try {
+ return innerDelete(getFileStatus(path), recursive);
+ } catch (FileNotFoundException e) {
+ LOG.debug("Couldn't delete {} - does not exist", path);
+ return false;
+ }
+ }
+
+ /**
+ * Delete an object. See {@link #delete(Path, boolean)}.
+ *
+ * @param status fileStatus object
+ * @param recursive if path is a directory and set to
+ * true, the directory is deleted else throws an exception. In
+ * case of a file the recursive can be set to either true or false.
+ * @return true if delete is successful else false.
+ * @throws IOException due to inability to delete a directory or file.
+ */
+ private boolean innerDelete(FileStatus status, boolean recursive)
+ throws IOException {
+ Path f = status.getPath();
+ String key = pathToKey(f);
+ if (status.isDirectory()) {
+ if (!recursive) {
+ FileStatus[] statuses = listStatus(status.getPath());
+ // Check whether it is an empty directory or not
+ if (statuses.length > 0) {
+ throw new IOException("Cannot remove directory " + f +
+ ": It is not empty!");
+ } else {
+ // Delete empty directory without '-r'
+ key = AliyunOSSUtils.maybeAddTrailingSlash(key);
+ store.deleteObject(key);
+ }
+ } else {
+ store.deleteDirs(key);
+ }
+ } else {
+ store.deleteObject(key);
+ }
+
+ createFakeDirectoryIfNecessary(f);
+ return true;
+ }
+
+ private void createFakeDirectoryIfNecessary(Path f) throws IOException {
+ String key = pathToKey(f);
+ if (StringUtils.isNotEmpty(key) && !exists(f)) {
+ LOG.debug("Creating new fake directory at {}", f);
+ mkdir(pathToKey(f.getParent()));
+ }
+ }
+
+ @Override
+ public FileStatus getFileStatus(Path path) throws IOException {
+ Path qualifiedPath = path.makeQualified(uri, workingDir);
+ String key = pathToKey(qualifiedPath);
+
+ // Root always exists
+ if (key.length() == 0) {
+ return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+ }
+
+ ObjectMetadata meta = store.getObjectMetadata(key);
+ // If key not found and key does not end with "/"
+ if (meta == null && !key.endsWith("/")) {
+ // In case of 'dir + "/"'
+ key += "/";
+ meta = store.getObjectMetadata(key);
+ }
+ if (meta == null) {
+ ObjectListing listing = store.listObjects(key, 1, null, false);
+ if (CollectionUtils.isNotEmpty(listing.getObjectSummaries()) ||
+ CollectionUtils.isNotEmpty(listing.getCommonPrefixes())) {
+ return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+ } else {
+ throw new FileNotFoundException(path + ": No such file or directory!");
+ }
+ } else if (objectRepresentsDirectory(key, meta.getContentLength())) {
+ return new FileStatus(0, true, 1, 0, 0, qualifiedPath);
+ } else {
+ return new FileStatus(meta.getContentLength(), false, 1,
+ getDefaultBlockSize(path), meta.getLastModified().getTime(),
+ qualifiedPath);
+ }
+ }
+
+ @Override
+ public String getScheme() {
+ return "oss";
+ }
+
+ @Override
+ public URI getUri() {
+ return uri;
+ }
+
+ @Override
+ public Path getWorkingDirectory() {
+ return workingDir;
+ }
+
+ @Deprecated
+ public long getDefaultBlockSize() {
+ return getConf().getLong(FS_OSS_BLOCK_SIZE_KEY, FS_OSS_BLOCK_SIZE_DEFAULT);
+ }
+
+ @Override
+ public String getCanonicalServiceName() {
+ // Does not support Token
+ return null;
+ }
+
+ /**
+ * Initialize new FileSystem.
+ *
+ * @param name the uri of the file system, including host, port, etc.
+ * @param conf configuration of the file system
+ * @throws IOException IO problems
+ */
+ public void initialize(URI name, Configuration conf) throws IOException {
+ super.initialize(name, conf);
+
+ uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority());
+ workingDir = new Path("/user",
+ System.getProperty("user.name")).makeQualified(uri, null);
+
+ store = new AliyunOSSFileSystemStore();
+ store.initialize(name, conf, statistics);
+ maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
+ setConf(conf);
+ }
+
+ /**
+ * Check if OSS object represents a directory.
+ *
+ * @param name object key
+ * @param size object content length
+ * @return true if object represents a directory
+ */
+ private boolean objectRepresentsDirectory(final String name,
+ final long size) {
+ return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L;
+ }
+
+ /**
+ * Turn a path (relative or otherwise) into an OSS key.
+ *
+ * @param path the path of the file.
+ * @return the key of the object that represents the file.
+ */
+ private String pathToKey(Path path) {
+ if (!path.isAbsolute()) {
+ path = new Path(workingDir, path);
+ }
+
+ return path.toUri().getPath().substring(1);
+ }
+
+ private Path keyToPath(String key) {
+ return new Path("/" + key);
+ }
+
+ @Override
+ public FileStatus[] listStatus(Path path) throws IOException {
+ String key = pathToKey(path);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("List status for path: " + path);
+ }
+
+ final List<FileStatus> result = new ArrayList<FileStatus>();
+ final FileStatus fileStatus = getFileStatus(path);
+
+ if (fileStatus.isDirectory()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("listStatus: doing listObjects for directory " + key);
+ }
+
+ ObjectListing objects = store.listObjects(key, maxKeys, null, false);
+ while (true) {
+ statistics.incrementReadOps(1);
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ String objKey = objectSummary.getKey();
+ if (objKey.equals(key + "/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring: " + objKey);
+ }
+ continue;
+ } else {
+ Path keyPath = keyToPath(objectSummary.getKey())
+ .makeQualified(uri, workingDir);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding: fi: " + keyPath);
+ }
+ result.add(new FileStatus(objectSummary.getSize(), false, 1,
+ getDefaultBlockSize(keyPath),
+ objectSummary.getLastModified().getTime(), keyPath));
+ }
+ }
+
+ for (String prefix : objects.getCommonPrefixes()) {
+ if (prefix.equals(key + "/")) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring: " + prefix);
+ }
+ continue;
+ } else {
+ Path keyPath = keyToPath(prefix).makeQualified(uri, workingDir);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding: rd: " + keyPath);
+ }
+ result.add(new FileStatus(0, true, 1, 0, 0, keyPath));
+ }
+ }
+
+ if (objects.isTruncated()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("listStatus: list truncated - getting next batch");
+ }
+ String nextMarker = objects.getNextMarker();
+ objects = store.listObjects(key, maxKeys, nextMarker, false);
+ statistics.incrementReadOps(1);
+ } else {
+ break;
+ }
+ }
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding: rd (not a dir): " + path);
+ }
+ result.add(fileStatus);
+ }
+
+ return result.toArray(new FileStatus[result.size()]);
+ }
+
+ /**
+ * Used to create an empty file that represents an empty directory.
+ *
+ * @param key directory path
+ * @return true if directory is successfully created
+ * @throws IOException
+ */
+ private boolean mkdir(final String key) throws IOException {
+ String dirName = key;
+ if (StringUtils.isNotEmpty(key)) {
+ if (!key.endsWith("/")) {
+ dirName += "/";
+ }
+ store.storeEmptyFile(dirName);
+ }
+ return true;
+ }
+
+ @Override
+ public boolean mkdirs(Path path, FsPermission permission)
+ throws IOException {
+ try {
+ FileStatus fileStatus = getFileStatus(path);
+
+ if (fileStatus.isDirectory()) {
+ return true;
+ } else {
+ throw new FileAlreadyExistsException("Path is a file: " + path);
+ }
+ } catch (FileNotFoundException e) {
+ validatePath(path);
+ String key = pathToKey(path);
+ return mkdir(key);
+ }
+ }
+
+ /**
+ * Check whether the path is a valid path.
+ *
+ * @param path the path to be checked.
+ * @throws IOException
+ */
+ private void validatePath(Path path) throws IOException {
+ Path fPart = path.getParent();
+ do {
+ try {
+ FileStatus fileStatus = getFileStatus(fPart);
+ if (fileStatus.isDirectory()) {
+ // If path exists and a directory, exit
+ break;
+ } else {
+ throw new FileAlreadyExistsException(String.format(
+ "Can't make directory for path '%s', it is a file.", fPart));
+ }
+ } catch (FileNotFoundException fnfe) {
+ }
+ fPart = fPart.getParent();
+ } while (fPart != null);
+ }
+
+ @Override
+ public FSDataInputStream open(Path path, int bufferSize) throws IOException {
+ final FileStatus fileStatus = getFileStatus(path);
+ if (fileStatus.isDirectory()) {
+ throw new FileNotFoundException("Can't open " + path +
+ " because it is a directory");
+ }
+
+ return new FSDataInputStream(new AliyunOSSInputStream(getConf(), store,
+ pathToKey(path), fileStatus.getLen(), statistics));
+ }
+
+ @Override
+ public boolean rename(Path srcPath, Path dstPath) throws IOException {
+ if (srcPath.isRoot()) {
+ // Cannot rename root of file system
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cannot rename the root of a filesystem");
+ }
+ return false;
+ }
+ Path parent = dstPath.getParent();
+ while (parent != null && !srcPath.equals(parent)) {
+ parent = parent.getParent();
+ }
+ if (parent != null) {
+ return false;
+ }
+ FileStatus srcStatus = getFileStatus(srcPath);
+ FileStatus dstStatus;
+ try {
+ dstStatus = getFileStatus(dstPath);
+ } catch (FileNotFoundException fnde) {
+ dstStatus = null;
+ }
+ if (dstStatus == null) {
+ // If dst doesn't exist, check whether dst dir exists or not
+ dstStatus = getFileStatus(dstPath.getParent());
+ if (!dstStatus.isDirectory()) {
+ throw new IOException(String.format(
+ "Failed to rename %s to %s, %s is a file", srcPath, dstPath,
+ dstPath.getParent()));
+ }
+ } else {
+ if (srcStatus.getPath().equals(dstStatus.getPath())) {
+ return !srcStatus.isDirectory();
+ } else if (dstStatus.isDirectory()) {
+ // If dst is a directory
+ dstPath = new Path(dstPath, srcPath.getName());
+ FileStatus[] statuses;
+ try {
+ statuses = listStatus(dstPath);
+ } catch (FileNotFoundException fnde) {
+ statuses = null;
+ }
+ if (statuses != null && statuses.length > 0) {
+ // If dst exists and not a directory / not empty
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists or not empty!",
+ srcPath, dstPath));
+ }
+ } else {
+ // If dst is not a directory
+ throw new FileAlreadyExistsException(String.format(
+ "Failed to rename %s to %s, file already exists!", srcPath,
+ dstPath));
+ }
+ }
+ if (srcStatus.isDirectory()) {
+ copyDirectory(srcPath, dstPath);
+ } else {
+ copyFile(srcPath, dstPath);
+ }
+
+ return srcPath.equals(dstPath) || delete(srcPath, true);
+ }
+
+ /**
+ * Copy file from source path to destination path.
+ * (the caller should make sure srcPath is a file and dstPath is valid)
+ *
+ * @param srcPath source path.
+ * @param dstPath destination path.
+ * @return true if file is successfully copied.
+ */
+ private boolean copyFile(Path srcPath, Path dstPath) {
+ String srcKey = pathToKey(srcPath);
+ String dstKey = pathToKey(dstPath);
+ return store.copyFile(srcKey, dstKey);
+ }
+
+ /**
+ * Copy a directory from source path to destination path.
+ * (the caller should make sure srcPath is a directory, and dstPath is valid)
+ *
+ * @param srcPath source path.
+ * @param dstPath destination path.
+ * @return true if directory is successfully copied.
+ */
+ private boolean copyDirectory(Path srcPath, Path dstPath) throws IOException {
+ String srcKey = AliyunOSSUtils
+ .maybeAddTrailingSlash(pathToKey(srcPath));
+ String dstKey = AliyunOSSUtils
+ .maybeAddTrailingSlash(pathToKey(dstPath));
+
+ if (dstKey.startsWith(srcKey)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Cannot rename a directory to a subdirectory of self");
+ }
+ return false;
+ }
+
+ store.storeEmptyFile(dstKey);
+ ObjectListing objects = store.listObjects(srcKey, maxKeys, null, true);
+ statistics.incrementReadOps(1);
+ // Copy files from src folder to dst
+ while (true) {
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ String newKey =
+ dstKey.concat(objectSummary.getKey().substring(srcKey.length()));
+ store.copyFile(objectSummary.getKey(), newKey);
+ }
+ if (objects.isTruncated()) {
+ String nextMarker = objects.getNextMarker();
+ objects = store.listObjects(srcKey, maxKeys, nextMarker, true);
+ statistics.incrementReadOps(1);
+ } else {
+ break;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public void setWorkingDirectory(Path dir) {
+ this.workingDir = dir;
+ }
+
+ public AliyunOSSFileSystemStore getStore() {
+ return store;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
new file mode 100644
index 0000000..9792a78
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -0,0 +1,516 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.ClientConfiguration;
+import com.aliyun.oss.ClientException;
+import com.aliyun.oss.OSSClient;
+import com.aliyun.oss.OSSException;
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.aliyun.oss.common.comm.Protocol;
+import com.aliyun.oss.model.AbortMultipartUploadRequest;
+import com.aliyun.oss.model.CannedAccessControlList;
+import com.aliyun.oss.model.CompleteMultipartUploadRequest;
+import com.aliyun.oss.model.CompleteMultipartUploadResult;
+import com.aliyun.oss.model.CopyObjectResult;
+import com.aliyun.oss.model.DeleteObjectsRequest;
+import com.aliyun.oss.model.GetObjectRequest;
+import com.aliyun.oss.model.InitiateMultipartUploadRequest;
+import com.aliyun.oss.model.InitiateMultipartUploadResult;
+import com.aliyun.oss.model.ListObjectsRequest;
+import com.aliyun.oss.model.ObjectMetadata;
+import com.aliyun.oss.model.ObjectListing;
+import com.aliyun.oss.model.OSSObjectSummary;
+import com.aliyun.oss.model.PartETag;
+import com.aliyun.oss.model.PutObjectResult;
+import com.aliyun.oss.model.UploadPartCopyRequest;
+import com.aliyun.oss.model.UploadPartCopyResult;
+import com.aliyun.oss.model.UploadPartRequest;
+import com.aliyun.oss.model.UploadPartResult;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Core implementation of Aliyun OSS Filesystem for Hadoop.
+ * Provides the bridging logic between Hadoop's abstract filesystem and
+ * Aliyun OSS.
+ */
+public class AliyunOSSFileSystemStore {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSFileSystemStore.class);
+ private FileSystem.Statistics statistics;
+ private OSSClient ossClient;
+ private String bucketName;
+ private long uploadPartSize;
+ private long multipartThreshold;
+ private long partSize;
+ private int maxKeys;
+ private String serverSideEncryptionAlgorithm;
+
+ public void initialize(URI uri, Configuration conf,
+ FileSystem.Statistics stat) throws IOException {
+ statistics = stat;
+ ClientConfiguration clientConf = new ClientConfiguration();
+ clientConf.setMaxConnections(conf.getInt(MAXIMUM_CONNECTIONS_KEY,
+ MAXIMUM_CONNECTIONS_DEFAULT));
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS_KEY,
+ SECURE_CONNECTIONS_DEFAULT);
+ clientConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
+ clientConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES_KEY,
+ MAX_ERROR_RETRIES_DEFAULT));
+ clientConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT_KEY,
+ ESTABLISH_TIMEOUT_DEFAULT));
+ clientConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT_KEY,
+ SOCKET_TIMEOUT_DEFAULT));
+
+ String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
+ int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
+ if (StringUtils.isNotEmpty(proxyHost)) {
+ clientConf.setProxyHost(proxyHost);
+ if (proxyPort >= 0) {
+ clientConf.setProxyPort(proxyPort);
+ } else {
+ if (secureConnections) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ clientConf.setProxyPort(443);
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ clientConf.setProxyPort(80);
+ }
+ }
+ String proxyUsername = conf.getTrimmed(PROXY_USERNAME_KEY);
+ String proxyPassword = conf.getTrimmed(PROXY_PASSWORD_KEY);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME_KEY + " or " +
+ PROXY_PASSWORD_KEY + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ clientConf.setProxyUsername(proxyUsername);
+ clientConf.setProxyPassword(proxyPassword);
+ clientConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN_KEY));
+ clientConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION_KEY));
+ } else if (proxyPort >= 0) {
+ String msg = "Proxy error: " + PROXY_PORT_KEY + " set without " +
+ PROXY_HOST_KEY;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+
+ String endPoint = conf.getTrimmed(ENDPOINT_KEY, "");
+ CredentialsProvider provider =
+ AliyunOSSUtils.getCredentialsProvider(conf);
+ ossClient = new OSSClient(endPoint, provider, clientConf);
+ uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
+ MULTIPART_UPLOAD_SIZE_DEFAULT);
+ multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
+ MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
+ partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
+ MULTIPART_UPLOAD_SIZE_DEFAULT);
+ if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
+ partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
+ }
+ serverSideEncryptionAlgorithm =
+ conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
+
+ if (uploadPartSize < 5 * 1024 * 1024) {
+ LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB");
+ uploadPartSize = 5 * 1024 * 1024;
+ }
+
+ if (multipartThreshold < 5 * 1024 * 1024) {
+ LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB");
+ multipartThreshold = 5 * 1024 * 1024;
+ }
+
+ if (multipartThreshold > 1024 * 1024 * 1024) {
+ LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be less than 1 GB");
+ multipartThreshold = 1024 * 1024 * 1024;
+ }
+
+ String cannedACLName = conf.get(CANNED_ACL_KEY, CANNED_ACL_DEFAULT);
+ if (StringUtils.isNotEmpty(cannedACLName)) {
+ CannedAccessControlList cannedACL =
+ CannedAccessControlList.valueOf(cannedACLName);
+ ossClient.setBucketAcl(bucketName, cannedACL);
+ }
+
+ maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
+ bucketName = uri.getHost();
+ }
+
+ /**
+ * Delete an object, and update write operation statistics.
+ *
+ * @param key key to blob to delete.
+ */
+ public void deleteObject(String key) {
+ ossClient.deleteObject(bucketName, key);
+ statistics.incrementWriteOps(1);
+ }
+
+ /**
+ * Delete a list of keys, and update write operation statistics.
+ *
+ * @param keysToDelete collection of keys to delete.
+ */
+ public void deleteObjects(List<String> keysToDelete) {
+ if (CollectionUtils.isNotEmpty(keysToDelete)) {
+ DeleteObjectsRequest deleteRequest =
+ new DeleteObjectsRequest(bucketName);
+ deleteRequest.setKeys(keysToDelete);
+ ossClient.deleteObjects(deleteRequest);
+ statistics.incrementWriteOps(keysToDelete.size());
+ }
+ }
+
+ /**
+ * Delete a directory from Aliyun OSS.
+ *
+ * @param key directory key to delete.
+ */
+ public void deleteDirs(String key) {
+ key = AliyunOSSUtils.maybeAddTrailingSlash(key);
+ ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+ listRequest.setPrefix(key);
+ listRequest.setDelimiter(null);
+ listRequest.setMaxKeys(maxKeys);
+
+ while (true) {
+ ObjectListing objects = ossClient.listObjects(listRequest);
+ statistics.incrementReadOps(1);
+ List<String> keysToDelete = new ArrayList<String>();
+ for (OSSObjectSummary objectSummary : objects.getObjectSummaries()) {
+ keysToDelete.add(objectSummary.getKey());
+ }
+ deleteObjects(keysToDelete);
+ if (objects.isTruncated()) {
+ listRequest.setMarker(objects.getNextMarker());
+ } else {
+ break;
+ }
+ }
+ }
+
+ /**
+ * Return metadata of a given object key.
+ *
+ * @param key object key.
+ * @return return null if key does not exist.
+ */
+ public ObjectMetadata getObjectMetadata(String key) {
+ try {
+ return ossClient.getObjectMetadata(bucketName, key);
+ } catch (OSSException osse) {
+ return null;
+ } finally {
+ statistics.incrementReadOps(1);
+ }
+ }
+
+ /**
+ * Upload an empty file as an OSS object, using single upload.
+ *
+ * @param key object key.
+ * @throws IOException if failed to upload object.
+ */
+ public void storeEmptyFile(String key) throws IOException {
+ ObjectMetadata dirMeta = new ObjectMetadata();
+ byte[] buffer = new byte[0];
+ ByteArrayInputStream in = new ByteArrayInputStream(buffer);
+ dirMeta.setContentLength(0);
+ try {
+ ossClient.putObject(bucketName, key, in, dirMeta);
+ } finally {
+ in.close();
+ }
+ }
+
+ /**
+ * Copy an object from source key to destination key.
+ *
+ * @param srcKey source key.
+ * @param dstKey destination key.
+ * @return true if file is successfully copied.
+ */
+ public boolean copyFile(String srcKey, String dstKey) {
+ ObjectMetadata objectMeta =
+ ossClient.getObjectMetadata(bucketName, srcKey);
+ long contentLength = objectMeta.getContentLength();
+ if (contentLength <= multipartThreshold) {
+ return singleCopy(srcKey, dstKey);
+ } else {
+ return multipartCopy(srcKey, contentLength, dstKey);
+ }
+ }
+
+ /**
+ * Use single copy to copy an OSS object.
+ * (The caller should make sure srcPath is a file and dstPath is valid)
+ *
+ * @param srcKey source key.
+ * @param dstKey destination key.
+ * @return true if object is successfully copied.
+ */
+ private boolean singleCopy(String srcKey, String dstKey) {
+ CopyObjectResult copyResult =
+ ossClient.copyObject(bucketName, srcKey, bucketName, dstKey);
+ LOG.debug(copyResult.getETag());
+ return true;
+ }
+
+ /**
+ * Use multipart copy to copy an OSS object.
+ * (The caller should make sure srcPath is a file and dstPath is valid)
+ *
+ * @param srcKey source key.
+ * @param contentLength data size of the object to copy.
+ * @param dstKey destination key.
+ * @return true if success, or false if upload is aborted.
+ */
+ private boolean multipartCopy(String srcKey, long contentLength,
+ String dstKey) {
+ long realPartSize =
+ AliyunOSSUtils.calculatePartSize(contentLength, uploadPartSize);
+ int partNum = (int) (contentLength / realPartSize);
+ if (contentLength % realPartSize != 0) {
+ partNum++;
+ }
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, dstKey);
+ ObjectMetadata meta = new ObjectMetadata();
+ if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ initiateMultipartUploadRequest.setObjectMetadata(meta);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ String uploadId = initiateMultipartUploadResult.getUploadId();
+ List<PartETag> partETags = new ArrayList<PartETag>();
+ try {
+ for (int i = 0; i < partNum; i++) {
+ long skipBytes = realPartSize * i;
+ long size = (realPartSize < contentLength - skipBytes) ?
+ realPartSize : contentLength - skipBytes;
+ UploadPartCopyRequest partCopyRequest = new UploadPartCopyRequest();
+ partCopyRequest.setSourceBucketName(bucketName);
+ partCopyRequest.setSourceKey(srcKey);
+ partCopyRequest.setBucketName(bucketName);
+ partCopyRequest.setKey(dstKey);
+ partCopyRequest.setUploadId(uploadId);
+ partCopyRequest.setPartSize(size);
+ partCopyRequest.setBeginIndex(skipBytes);
+ partCopyRequest.setPartNumber(i + 1);
+ UploadPartCopyResult partCopyResult =
+ ossClient.uploadPartCopy(partCopyRequest);
+ statistics.incrementWriteOps(1);
+ partETags.add(partCopyResult.getPartETag());
+ }
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, dstKey,
+ uploadId, partETags);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ LOG.debug(completeMultipartUploadResult.getETag());
+ return true;
+ } catch (OSSException | ClientException e) {
+ AbortMultipartUploadRequest abortMultipartUploadRequest =
+ new AbortMultipartUploadRequest(bucketName, dstKey, uploadId);
+ ossClient.abortMultipartUpload(abortMultipartUploadRequest);
+ return false;
+ }
+ }
+
+ /**
+ * Upload a file as an OSS object, using single upload.
+ *
+ * @param key object key.
+ * @param file local file to upload.
+ * @throws IOException if failed to upload object.
+ */
+ public void uploadObject(String key, File file) throws IOException {
+ File object = file.getAbsoluteFile();
+ FileInputStream fis = new FileInputStream(object);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(object.length());
+ if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ try {
+ PutObjectResult result = ossClient.putObject(bucketName, key, fis, meta);
+ LOG.debug(result.getETag());
+ statistics.incrementWriteOps(1);
+ } finally {
+ fis.close();
+ }
+ }
+
+ /**
+ * Upload a file as an OSS object, using multipart upload.
+ *
+ * @param key object key.
+ * @param file local file to upload.
+ * @throws IOException if failed to upload object.
+ */
+ public void multipartUploadObject(String key, File file) throws IOException {
+ File object = file.getAbsoluteFile();
+ long dataLen = object.length();
+ long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
+ int partNum = (int) (dataLen / realPartSize);
+ if (dataLen % realPartSize != 0) {
+ partNum += 1;
+ }
+
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, key);
+ ObjectMetadata meta = new ObjectMetadata();
+ if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
+ meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
+ }
+ initiateMultipartUploadRequest.setObjectMetadata(meta);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ List<PartETag> partETags = new ArrayList<PartETag>();
+ String uploadId = initiateMultipartUploadResult.getUploadId();
+
+ try {
+ for (int i = 0; i < partNum; i++) {
+ // TODO: Optimize this, avoid opening the object multiple times
+ FileInputStream fis = new FileInputStream(object);
+ try {
+ long skipBytes = realPartSize * i;
+ AliyunOSSUtils.skipFully(fis, skipBytes);
+ long size = (realPartSize < dataLen - skipBytes) ?
+ realPartSize : dataLen - skipBytes;
+ UploadPartRequest uploadPartRequest = new UploadPartRequest();
+ uploadPartRequest.setBucketName(bucketName);
+ uploadPartRequest.setKey(key);
+ uploadPartRequest.setUploadId(uploadId);
+ uploadPartRequest.setInputStream(fis);
+ uploadPartRequest.setPartSize(size);
+ uploadPartRequest.setPartNumber(i + 1);
+ UploadPartResult uploadPartResult =
+ ossClient.uploadPart(uploadPartRequest);
+ statistics.incrementWriteOps(1);
+ partETags.add(uploadPartResult.getPartETag());
+ } finally {
+ fis.close();
+ }
+ }
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, key,
+ uploadId, partETags);
+ CompleteMultipartUploadResult completeMultipartUploadResult =
+ ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ LOG.debug(completeMultipartUploadResult.getETag());
+ } catch (OSSException | ClientException e) {
+ AbortMultipartUploadRequest abortMultipartUploadRequest =
+ new AbortMultipartUploadRequest(bucketName, key, uploadId);
+ ossClient.abortMultipartUpload(abortMultipartUploadRequest);
+ }
+ }
+
+ /**
+ * list objects.
+ *
+ * @param prefix prefix.
+ * @param maxListingLength max no. of entries
+ * @param marker last key in any previous search.
+ * @param recursive whether to list directory recursively.
+ * @return a list of matches.
+ */
+ public ObjectListing listObjects(String prefix, int maxListingLength,
+ String marker, boolean recursive) {
+ String delimiter = recursive ? null : "/";
+ prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix);
+ ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
+ listRequest.setPrefix(prefix);
+ listRequest.setDelimiter(delimiter);
+ listRequest.setMaxKeys(maxListingLength);
+ listRequest.setMarker(marker);
+
+ ObjectListing listing = ossClient.listObjects(listRequest);
+ statistics.incrementReadOps(1);
+ return listing;
+ }
+
+ /**
+ * Retrieve a part of an object.
+ *
+ * @param key the object name that is being retrieved from the Aliyun OSS.
+ * @param byteStart start position.
+ * @param byteEnd end position.
+ * @return This method returns null if the key is not found.
+ */
+ public InputStream retrieve(String key, long byteStart, long byteEnd) {
+ try {
+ GetObjectRequest request = new GetObjectRequest(bucketName, key);
+ request.setRange(byteStart, byteEnd);
+ return ossClient.getObject(request).getObjectContent();
+ } catch (OSSException | ClientException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Close OSS client properly.
+ */
+ public void close() {
+ if (ossClient != null) {
+ ossClient.shutdown();
+ ossClient = null;
+ }
+ }
+
+ /**
+ * Clean up all objects matching the prefix.
+ *
+ * @param prefix Aliyun OSS object prefix.
+ */
+ public void purge(String prefix) {
+ String key;
+ try {
+ ObjectListing objects = listObjects(prefix, maxKeys, null, true);
+ for (OSSObjectSummary object : objects.getObjectSummaries()) {
+ key = object.getKey();
+ ossClient.deleteObject(bucketName, key);
+ }
+
+ for (String dir: objects.getCommonPrefixes()) {
+ deleteDirs(dir);
+ }
+ } catch (OSSException | ClientException e) {
+ LOG.error("Failed to purge " + prefix);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
new file mode 100644
index 0000000..b87a3a7
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSInputStream.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSExceptionMessages;
+import org.apache.hadoop.fs.FSInputStream;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * The input stream for OSS blob system.
+ * The class uses multi-part downloading to read data from the object content
+ * stream.
+ */
+public class AliyunOSSInputStream extends FSInputStream {
+ public static final Log LOG = LogFactory.getLog(AliyunOSSInputStream.class);
+ private final long downloadPartSize;
+ private AliyunOSSFileSystemStore store;
+ private final String key;
+ private Statistics statistics;
+ private boolean closed;
+ private InputStream wrappedStream = null;
+ private long contentLength;
+ private long position;
+ private long partRemaining;
+
+ public AliyunOSSInputStream(Configuration conf,
+ AliyunOSSFileSystemStore store, String key, Long contentLength,
+ Statistics statistics) throws IOException {
+ this.store = store;
+ this.key = key;
+ this.statistics = statistics;
+ this.contentLength = contentLength;
+ downloadPartSize = conf.getLong(MULTIPART_DOWNLOAD_SIZE_KEY,
+ MULTIPART_DOWNLOAD_SIZE_DEFAULT);
+ reopen(0);
+ closed = false;
+ }
+
+ /**
+ * Reopen the wrapped stream at give position, by seeking for
+ * data of a part length from object content stream.
+ *
+ * @param pos position from start of a file
+ * @throws IOException if failed to reopen
+ */
+ private synchronized void reopen(long pos) throws IOException {
+ long partSize;
+
+ if (pos < 0) {
+ throw new EOFException("Cannot seek at negative position:" + pos);
+ } else if (pos > contentLength) {
+ throw new EOFException("Cannot seek after EOF, contentLength:" +
+ contentLength + " position:" + pos);
+ } else if (pos + downloadPartSize > contentLength) {
+ partSize = contentLength - pos;
+ } else {
+ partSize = downloadPartSize;
+ }
+
+ if (wrappedStream != null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Aborting old stream to open at pos " + pos);
+ }
+ wrappedStream.close();
+ }
+
+ wrappedStream = store.retrieve(key, pos, pos + partSize -1);
+ if (wrappedStream == null) {
+ throw new IOException("Null IO stream");
+ }
+ position = pos;
+ partRemaining = partSize;
+ }
+
+ @Override
+ public synchronized int read() throws IOException {
+ checkNotClosed();
+
+ if (partRemaining <= 0 && position < contentLength) {
+ reopen(position);
+ }
+
+ int tries = MAX_RETRIES;
+ boolean retry;
+ int byteRead = -1;
+ do {
+ retry = false;
+ try {
+ byteRead = wrappedStream.read();
+ } catch (Exception e) {
+ handleReadException(e, --tries);
+ retry = true;
+ }
+ } while (retry);
+ if (byteRead >= 0) {
+ position++;
+ partRemaining--;
+ }
+
+ if (statistics != null && byteRead >= 0) {
+ statistics.incrementBytesRead(1);
+ }
+ return byteRead;
+ }
+
+
+ /**
+ * Verify that the input stream is open. Non blocking; this gives
+ * the last state of the volatile {@link #closed} field.
+ *
+ * @throws IOException if the connection is closed.
+ */
+ private void checkNotClosed() throws IOException {
+ if (closed) {
+ throw new IOException(FSExceptionMessages.STREAM_IS_CLOSED);
+ }
+ }
+
+ @Override
+ public synchronized int read(byte[] buf, int off, int len)
+ throws IOException {
+ checkNotClosed();
+
+ if (buf == null) {
+ throw new NullPointerException();
+ } else if (off < 0 || len < 0 || len > buf.length - off) {
+ throw new IndexOutOfBoundsException();
+ } else if (len == 0) {
+ return 0;
+ }
+
+ int bytesRead = 0;
+ // Not EOF, and read not done
+ while (position < contentLength && bytesRead < len) {
+ if (partRemaining == 0) {
+ reopen(position);
+ }
+
+ int tries = MAX_RETRIES;
+ boolean retry;
+ int bytes = -1;
+ do {
+ retry = false;
+ try {
+ bytes = wrappedStream.read(buf, off + bytesRead, len - bytesRead);
+ } catch (Exception e) {
+ handleReadException(e, --tries);
+ retry = true;
+ }
+ } while (retry);
+
+ if (bytes > 0) {
+ bytesRead += bytes;
+ position += bytes;
+ partRemaining -= bytes;
+ } else if (partRemaining != 0) {
+ throw new IOException("Failed to read from stream. Remaining:" +
+ partRemaining);
+ }
+ }
+
+ if (statistics != null && bytesRead > 0) {
+ statistics.incrementBytesRead(bytesRead);
+ }
+
+ // Read nothing, but attempt to read something
+ if (bytesRead == 0 && len > 0) {
+ return -1;
+ } else {
+ return bytesRead;
+ }
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ if (wrappedStream != null) {
+ wrappedStream.close();
+ }
+ }
+
+ @Override
+ public synchronized int available() throws IOException {
+ checkNotClosed();
+
+ long remaining = contentLength - position;
+ if (remaining > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ }
+ return (int)remaining;
+ }
+
+ @Override
+ public synchronized void seek(long pos) throws IOException {
+ checkNotClosed();
+ if (position == pos) {
+ return;
+ } else if (pos > position && pos < position + partRemaining) {
+ AliyunOSSUtils.skipFully(wrappedStream, pos - position);
+ position = pos;
+ } else {
+ reopen(pos);
+ }
+ }
+
+ @Override
+ public synchronized long getPos() throws IOException {
+ checkNotClosed();
+ return position;
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ checkNotClosed();
+ return false;
+ }
+
+ private void handleReadException(Exception e, int tries) throws IOException{
+ if (tries == 0) {
+ throw new IOException(e);
+ }
+
+ LOG.warn("Some exceptions occurred in oss connection, try to reopen oss" +
+ " connection at position '" + position + "', " + e.getMessage());
+ try {
+ Thread.sleep(100);
+ } catch (InterruptedException e2) {
+ LOG.warn(e2.getMessage());
+ }
+ reopen(position);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
new file mode 100644
index 0000000..c75ee18
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.util.Progressable;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * The output stream for OSS blob system.
+ * Data will be buffered on local disk, then uploaded to OSS in
+ * {@link #close()} method.
+ */
+public class AliyunOSSOutputStream extends OutputStream {
+ public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class);
+ private AliyunOSSFileSystemStore store;
+ private final String key;
+ private Statistics statistics;
+ private Progressable progress;
+ private long partSizeThreshold;
+ private LocalDirAllocator dirAlloc;
+ private boolean closed;
+ private File tmpFile;
+ private BufferedOutputStream backupStream;
+
+ public AliyunOSSOutputStream(Configuration conf,
+ AliyunOSSFileSystemStore store, String key, Progressable progress,
+ Statistics statistics) throws IOException {
+ this.store = store;
+ this.key = key;
+ // The caller cann't get any progress information
+ this.progress = progress;
+ this.statistics = statistics;
+ partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
+ MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
+
+ if (conf.get(BUFFER_DIR_KEY) == null) {
+ conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
+ }
+ dirAlloc = new LocalDirAllocator(BUFFER_DIR_KEY);
+
+ tmpFile = dirAlloc.createTmpFileForWrite("output-",
+ LocalDirAllocator.SIZE_UNKNOWN, conf);
+ backupStream = new BufferedOutputStream(new FileOutputStream(tmpFile));
+ closed = false;
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ closed = true;
+ if (backupStream != null) {
+ backupStream.close();
+ }
+ long dataLen = tmpFile.length();
+ try {
+ if (dataLen <= partSizeThreshold) {
+ store.uploadObject(key, tmpFile);
+ } else {
+ store.multipartUploadObject(key, tmpFile);
+ }
+ } finally {
+ if (!tmpFile.delete()) {
+ LOG.warn("Can not delete file: " + tmpFile);
+ }
+ }
+ }
+
+
+
+ @Override
+ public synchronized void flush() throws IOException {
+ backupStream.flush();
+ }
+
+ @Override
+ public synchronized void write(int b) throws IOException {
+ backupStream.write(b);
+ statistics.incrementBytesWritten(1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
new file mode 100644
index 0000000..cae9749
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -0,0 +1,167 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import com.aliyun.oss.common.auth.CredentialsProvider;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.ProviderUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Utility methods for Aliyun OSS code.
+ */
+final public class AliyunOSSUtils {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSUtils.class);
+
+ private AliyunOSSUtils() {
+ }
+
+ /**
+ * Used to get password from configuration.
+ *
+ * @param conf configuration that contains password information
+ * @param key the key of the password
+ * @return the value for the key
+ * @throws IOException if failed to get password from configuration
+ */
+ public static String getValueWithKey(Configuration conf, String key)
+ throws IOException {
+ try {
+ final char[] pass = conf.getPassword(key);
+ if (pass != null) {
+ return (new String(pass)).trim();
+ } else {
+ return "";
+ }
+ } catch (IOException ioe) {
+ throw new IOException("Cannot find password option " + key, ioe);
+ }
+ }
+
+ /**
+ * Skip the requested number of bytes or fail if there are no enough bytes
+ * left. This allows for the possibility that {@link InputStream#skip(long)}
+ * may not skip as many bytes as requested (most likely because of reaching
+ * EOF).
+ *
+ * @param is the input stream to skip.
+ * @param n the number of bytes to skip.
+ * @throws IOException thrown when skipped less number of bytes.
+ */
+ public static void skipFully(InputStream is, long n) throws IOException {
+ long total = 0;
+ long cur = 0;
+
+ do {
+ cur = is.skip(n - total);
+ total += cur;
+ } while((total < n) && (cur > 0));
+
+ if (total < n) {
+ throw new IOException("Failed to skip " + n + " bytes, possibly due " +
+ "to EOF.");
+ }
+ }
+
+ /**
+ * Calculate a proper size of multipart piece. If <code>minPartSize</code>
+ * is too small, the number of multipart pieces may exceed the limit of
+ * {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}.
+ *
+ * @param contentLength the size of file.
+ * @param minPartSize the minimum size of multipart piece.
+ * @return a revisional size of multipart piece.
+ */
+ public static long calculatePartSize(long contentLength, long minPartSize) {
+ long tmpPartSize = contentLength / MULTIPART_UPLOAD_PART_NUM_LIMIT + 1;
+ return Math.max(minPartSize, tmpPartSize);
+ }
+
+ /**
+ * Create credential provider specified by configuration, or create default
+ * credential provider if not specified.
+ *
+ * @param conf configuration
+ * @return a credential provider
+ * @throws IOException on any problem. Class construction issues may be
+ * nested inside the IOE.
+ */
+ public static CredentialsProvider getCredentialsProvider(Configuration conf)
+ throws IOException {
+ CredentialsProvider credentials;
+
+ String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY);
+ if (StringUtils.isEmpty(className)) {
+ Configuration newConf =
+ ProviderUtils.excludeIncompatibleCredentialProviders(conf,
+ AliyunOSSFileSystem.class);
+ credentials = new AliyunCredentialsProvider(newConf);
+ } else {
+ try {
+ LOG.debug("Credential provider class is:" + className);
+ Class<?> credClass = Class.forName(className);
+ try {
+ credentials =
+ (CredentialsProvider)credClass.getDeclaredConstructor(
+ Configuration.class).newInstance(conf);
+ } catch (NoSuchMethodException | SecurityException e) {
+ credentials =
+ (CredentialsProvider)credClass.getDeclaredConstructor()
+ .newInstance();
+ }
+ } catch (ClassNotFoundException e) {
+ throw new IOException(className + " not found.", e);
+ } catch (NoSuchMethodException | SecurityException e) {
+ throw new IOException(String.format("%s constructor exception. A " +
+ "class specified in %s must provide an accessible constructor " +
+ "accepting URI and Configuration, or an accessible default " +
+ "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY),
+ e);
+ } catch (ReflectiveOperationException | IllegalArgumentException e) {
+ throw new IOException(className + " instantiation exception.", e);
+ }
+ }
+
+ return credentials;
+ }
+
+ /**
+ * Turns a path (relative or otherwise) into an OSS key, adding a trailing
+ * "/" if the path is not the root <i>and</i> does not already have a "/"
+ * at the end.
+ *
+ * @param key OSS key or ""
+ * @return the with a trailing "/", or, if it is the root key, "".
+ */
+ public static String maybeAddTrailingSlash(String key) {
+ if (StringUtils.isNotEmpty(key) && !key.endsWith("/")) {
+ return key + '/';
+ } else {
+ return key;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
new file mode 100644
index 0000000..04a2ccd
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -0,0 +1,113 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+/**
+ * ALL configuration constants for OSS filesystem.
+ */
+public final class Constants {
+
+ private Constants() {
+ }
+
+ // Class of credential provider
+ public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY =
+ "fs.oss.credentials.provider";
+
+ // OSS access verification
+ public static final String ACCESS_KEY_ID = "fs.oss.accessKeyId";
+ public static final String ACCESS_KEY_SECRET = "fs.oss.accessKeySecret";
+ public static final String SECURITY_TOKEN = "fs.oss.securityToken";
+
+ // Number of simultaneous connections to oss
+ public static final String MAXIMUM_CONNECTIONS_KEY =
+ "fs.oss.connection.maximum";
+ public static final int MAXIMUM_CONNECTIONS_DEFAULT = 32;
+
+ // Connect to oss over ssl
+ public static final String SECURE_CONNECTIONS_KEY =
+ "fs.oss.connection.secure.enabled";
+ public static final boolean SECURE_CONNECTIONS_DEFAULT = true;
+
+ // Use a custom endpoint
+ public static final String ENDPOINT_KEY = "fs.oss.endpoint";
+
+ // Connect to oss through a proxy server
+ public static final String PROXY_HOST_KEY = "fs.oss.proxy.host";
+ public static final String PROXY_PORT_KEY = "fs.oss.proxy.port";
+ public static final String PROXY_USERNAME_KEY = "fs.oss.proxy.username";
+ public static final String PROXY_PASSWORD_KEY = "fs.oss.proxy.password";
+ public static final String PROXY_DOMAIN_KEY = "fs.oss.proxy.domain";
+ public static final String PROXY_WORKSTATION_KEY =
+ "fs.oss.proxy.workstation";
+
+ // Number of times we should retry errors
+ public static final String MAX_ERROR_RETRIES_KEY = "fs.oss.attempts.maximum";
+ public static final int MAX_ERROR_RETRIES_DEFAULT = 20;
+
+ // Time until we give up trying to establish a connection to oss
+ public static final String ESTABLISH_TIMEOUT_KEY =
+ "fs.oss.connection.establish.timeout";
+ public static final int ESTABLISH_TIMEOUT_DEFAULT = 50000;
+
+ // Time until we give up on a connection to oss
+ public static final String SOCKET_TIMEOUT_KEY = "fs.oss.connection.timeout";
+ public static final int SOCKET_TIMEOUT_DEFAULT = 200000;
+
+ // Number of records to get while paging through a directory listing
+ public static final String MAX_PAGING_KEYS_KEY = "fs.oss.paging.maximum";
+ public static final int MAX_PAGING_KEYS_DEFAULT = 1000;
+
+ // Size of each of or multipart pieces in bytes
+ public static final String MULTIPART_UPLOAD_SIZE_KEY =
+ "fs.oss.multipart.upload.size";
+
+ public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024;
+ public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000;
+
+ // Minimum size in bytes before we start a multipart uploads or copy
+ public static final String MIN_MULTIPART_UPLOAD_THRESHOLD_KEY =
+ "fs.oss.multipart.upload.threshold";
+ public static final long MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT =
+ 20 * 1024 * 1024;
+
+ public static final String MULTIPART_DOWNLOAD_SIZE_KEY =
+ "fs.oss.multipart.download.size";
+
+ public static final long MULTIPART_DOWNLOAD_SIZE_DEFAULT = 100 * 1024;
+
+ // Comma separated list of directories
+ public static final String BUFFER_DIR_KEY = "fs.oss.buffer.dir";
+
+ // private | public-read | public-read-write
+ public static final String CANNED_ACL_KEY = "fs.oss.acl.default";
+ public static final String CANNED_ACL_DEFAULT = "";
+
+ // OSS server-side encryption
+ public static final String SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY =
+ "fs.oss.server-side-encryption-algorithm";
+
+ public static final String FS_OSS_BLOCK_SIZE_KEY = "fs.oss.block.size";
+ public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024;
+ public static final String FS_OSS = "oss";
+
+ public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L;
+ public static final int MAX_RETRIES = 10;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java
new file mode 100644
index 0000000..234567b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/package-info.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Aliyun OSS Filesystem.
+ */
+package org.apache.hadoop.fs.aliyun.oss;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: HDFS-10877. Make
RemoteEditLogManifest.committedTxnId optional in Protocol Buffers.
Contributed by Sean Mackrory.
Posted by ji...@apache.org.
HDFS-10877. Make RemoteEditLogManifest.committedTxnId optional in Protocol Buffers. Contributed by Sean Mackrory.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40acacee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40acacee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40acacee
Branch: refs/heads/MAPREDUCE-6608
Commit: 40acacee085494ca52205d37449a46c058d5d325
Parents: 8d619b4
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Sep 22 11:43:11 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Sep 22 11:43:11 2016 -0700
----------------------------------------------------------------------
.../server/protocol/RemoteEditLogManifest.java | 7 ++++-
.../hadoop-hdfs/src/main/proto/HdfsServer.proto | 2 +-
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 30 ++++++++++++++------
3 files changed, 28 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40acacee/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
index 686f7c2..8252b3b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/RemoteEditLogManifest.java
@@ -22,6 +22,7 @@ import java.util.List;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
/**
* An enumeration of logs available on a remote NameNode.
@@ -30,11 +31,15 @@ public class RemoteEditLogManifest {
private List<RemoteEditLog> logs;
- private long committedTxnId = -1;
+ private long committedTxnId = HdfsServerConstants.INVALID_TXID;
public RemoteEditLogManifest() {
}
+ public RemoteEditLogManifest(List<RemoteEditLog> logs) {
+ this(logs, HdfsServerConstants.INVALID_TXID);
+ }
+
public RemoteEditLogManifest(List<RemoteEditLog> logs, long committedTxnId) {
this.logs = logs;
this.committedTxnId = committedTxnId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40acacee/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
index e87dc95..910e03b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
@@ -88,7 +88,7 @@ message RemoteEditLogProto {
*/
message RemoteEditLogManifestProto {
repeated RemoteEditLogProto logs = 1;
- required uint64 committedTxnId = 2;
+ optional uint64 committedTxnId = 2;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40acacee/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index af17756..4072071 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
@@ -323,26 +324,37 @@ public class TestPBHelper {
RemoteEditLog l1 = PBHelper.convert(lProto);
compare(l, l1);
}
-
- @Test
- public void testConvertRemoteEditLogManifest() {
- List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
- logs.add(new RemoteEditLog(1, 10));
- logs.add(new RemoteEditLog(11, 20));
- RemoteEditLogManifest m = new RemoteEditLogManifest(logs, 20);
+
+ private void convertAndCheckRemoteEditLogManifest(RemoteEditLogManifest m,
+ List<RemoteEditLog> logs,
+ long committedTxnId) {
RemoteEditLogManifestProto mProto = PBHelper.convert(m);
RemoteEditLogManifest m1 = PBHelper.convert(mProto);
-
+
List<RemoteEditLog> logs1 = m1.getLogs();
assertEquals(logs.size(), logs1.size());
for (int i = 0; i < logs.size(); i++) {
compare(logs.get(i), logs1.get(i));
}
+ assertEquals(committedTxnId, m.getCommittedTxnId());
+ }
+
+ @Test
+ public void testConvertRemoteEditLogManifest() {
+ List<RemoteEditLog> logs = new ArrayList<RemoteEditLog>();
+ logs.add(new RemoteEditLog(1, 10));
+ logs.add(new RemoteEditLog(11, 20));
+
+ convertAndCheckRemoteEditLogManifest(new RemoteEditLogManifest(logs, 20),
+ logs, 20);
+ convertAndCheckRemoteEditLogManifest(new RemoteEditLogManifest(logs),
+ logs, HdfsServerConstants.INVALID_TXID);
}
+
public ExtendedBlock getExtendedBlock() {
return getExtendedBlock(1);
}
-
+
public ExtendedBlock getExtendedBlock(long blkid) {
return new ExtendedBlock("bpid", blkid, 100, 2);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: HDFS-10843. Update space quota
when a UC block is completed rather than committed. Contributed by Erik
Krogen.
Posted by ji...@apache.org.
HDFS-10843. Update space quota when a UC block is completed rather than committed. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a5bb88c8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a5bb88c8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a5bb88c8
Branch: refs/heads/MAPREDUCE-6608
Commit: a5bb88c8e0fd4bd19b6d377fecbe1d2d441514f6
Parents: bbdf350
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Sep 22 13:57:37 2016 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Sep 23 10:37:46 2016 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 39 ++-
.../hdfs/server/namenode/FSDirAttrOp.java | 12 +-
.../hdfs/server/namenode/FSDirectory.java | 47 +++
.../hdfs/server/namenode/FSNamesystem.java | 39 +--
.../hadoop/hdfs/server/namenode/Namesystem.java | 2 +
.../namenode/TestDiskspaceQuotaUpdate.java | 301 ++++++++++++++-----
6 files changed, 313 insertions(+), 127 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5bb88c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 3a12d74..886984a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HAContext;
@@ -786,12 +787,13 @@ public class BlockManager implements BlockStatsMXBean {
*
* @param bc block collection
* @param commitBlock - contains client reported block length and generation
+ * @param iip - INodes in path to bc
* @return true if the last block is changed to committed state.
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
public boolean commitOrCompleteLastBlock(BlockCollection bc,
- Block commitBlock) throws IOException {
+ Block commitBlock, INodesInPath iip) throws IOException {
if(commitBlock == null)
return false; // not committing, this is a block allocation retry
BlockInfo lastBlock = bc.getLastBlock();
@@ -811,7 +813,7 @@ public class BlockManager implements BlockStatsMXBean {
if (committed) {
addExpectedReplicasToPending(lastBlock);
}
- completeBlock(lastBlock, false);
+ completeBlock(lastBlock, iip, false);
}
return committed;
}
@@ -841,11 +843,15 @@ public class BlockManager implements BlockStatsMXBean {
/**
* Convert a specified block of the file to a complete block.
+ * @param curBlock - block to be completed
+ * @param iip - INodes in path to file containing curBlock; if null,
+ * this will be resolved internally
+ * @param force - force completion of the block
* @throws IOException if the block does not have at least a minimal number
* of replicas reported from data-nodes.
*/
- private void completeBlock(BlockInfo curBlock, boolean force)
- throws IOException {
+ private void completeBlock(BlockInfo curBlock, INodesInPath iip,
+ boolean force) throws IOException {
if (curBlock.isComplete()) {
return;
}
@@ -860,7 +866,8 @@ public class BlockManager implements BlockStatsMXBean {
"Cannot complete block: block has not been COMMITTED by the client");
}
- curBlock.convertToCompleteBlock();
+ convertToCompleteBlock(curBlock, iip);
+
// Since safe-mode only counts complete blocks, and we now have
// one more complete block, we need to adjust the total up, and
// also count it as safe, if we have at least the minimum replica
@@ -875,13 +882,29 @@ public class BlockManager implements BlockStatsMXBean {
}
/**
+ * Convert a specified block of the file to a complete block.
+ * Skips validity checking and safe mode block total updates; use
+ * {@link BlockManager#completeBlock} to include these.
+ * @param curBlock - block to be completed
+ * @param iip - INodes in path to file containing curBlock; if null,
+ * this will be resolved internally
+ * @throws IOException if the block does not have at least a minimal number
+ * of replicas reported from data-nodes.
+ */
+ private void convertToCompleteBlock(BlockInfo curBlock, INodesInPath iip)
+ throws IOException {
+ curBlock.convertToCompleteBlock();
+ namesystem.getFSDirectory().updateSpaceForCompleteBlock(curBlock, iip);
+ }
+
+ /**
* Force the given block in the given file to be marked as complete,
* regardless of whether enough replicas are present. This is necessary
* when tailing edit logs as a Standby.
*/
public void forceCompleteBlock(final BlockInfo block) throws IOException {
block.commitBlock(block);
- completeBlock(block, true);
+ completeBlock(block, null, true);
}
/**
@@ -2910,7 +2933,7 @@ public class BlockManager implements BlockStatsMXBean {
int numCurrentReplica = countLiveNodes(storedBlock);
if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED
&& hasMinStorage(storedBlock, numCurrentReplica)) {
- completeBlock(storedBlock, false);
+ completeBlock(storedBlock, null, false);
} else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that.
@@ -2985,7 +3008,7 @@ public class BlockManager implements BlockStatsMXBean {
if(storedBlock.getBlockUCState() == BlockUCState.COMMITTED &&
hasMinStorage(storedBlock, numLiveReplicas)) {
addExpectedReplicasToPending(storedBlock);
- completeBlock(storedBlock, false);
+ completeBlock(storedBlock, null, false);
} else if (storedBlock.isComplete() && result == AddBlockResult.ADDED) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5bb88c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index e19341c..4c5ecb1d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -418,9 +418,9 @@ public class FSDirAttrOp {
// Make sure the directory has sufficient quotas
short oldBR = file.getPreferredBlockReplication();
+ long size = file.computeFileSize(true, true);
// Ensure the quota does not exceed
if (oldBR < replication) {
- long size = file.computeFileSize(true, true);
fsd.updateCount(iip, 0L, size, oldBR, replication, true);
}
@@ -428,14 +428,10 @@ public class FSDirAttrOp {
short targetReplication = (short) Math.max(
replication, file.getPreferredBlockReplication());
+ if (oldBR > replication) {
+ fsd.updateCount(iip, 0L, size, oldBR, targetReplication, true);
+ }
for (BlockInfo b : file.getBlocks()) {
- if (oldBR == targetReplication) {
- continue;
- }
- if (oldBR > replication) {
- fsd.updateCount(iip, 0L, b.getNumBytes(), oldBR, targetReplication,
- true);
- }
bm.setReplication(oldBR, targetReplication, b);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5bb88c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c913c23..2a3cabb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -47,6 +48,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
@@ -930,6 +932,51 @@ public class FSDirectory implements Closeable {
}
}
+ /**
+ * Update the cached quota space for a block that is being completed.
+ * Must only be called once, as the block is being completed.
+ * @param completeBlk - Completed block for which to update space
+ * @param inodes - INodes in path to file containing completeBlk; if null
+ * this will be resolved internally
+ */
+ public void updateSpaceForCompleteBlock(BlockInfo completeBlk,
+ INodesInPath inodes) throws IOException {
+ assert namesystem.hasWriteLock();
+ INodesInPath iip = inodes != null ? inodes :
+ INodesInPath.fromINode(namesystem.getBlockCollection(completeBlk));
+ INodeFile fileINode = iip.getLastINode().asFile();
+ // Adjust disk space consumption if required
+ final long diff;
+ final short replicationFactor;
+ if (fileINode.isStriped()) {
+ final ErasureCodingPolicy ecPolicy =
+ FSDirErasureCodingOp.getErasureCodingPolicy(namesystem, iip);
+ final short numDataUnits = (short) ecPolicy.getNumDataUnits();
+ final short numParityUnits = (short) ecPolicy.getNumParityUnits();
+
+ final long numBlocks = numDataUnits + numParityUnits;
+ final long fullBlockGroupSize =
+ fileINode.getPreferredBlockSize() * numBlocks;
+
+ final BlockInfoStriped striped =
+ new BlockInfoStriped(completeBlk, ecPolicy);
+ final long actualBlockGroupSize = striped.spaceConsumed();
+
+ diff = fullBlockGroupSize - actualBlockGroupSize;
+ replicationFactor = (short) 1;
+ } else {
+ diff = fileINode.getPreferredBlockSize() - completeBlk.getNumBytes();
+ replicationFactor = fileINode.getFileReplication();
+ }
+ if (diff > 0) {
+ try {
+ updateSpaceConsumed(iip, 0, -diff, replicationFactor);
+ } catch (IOException e) {
+ LOG.warn("Unexpected exception while updating disk space.", e);
+ }
+ }
+ }
+
public EnumCounters<StorageType> getStorageTypeDeltas(byte storagePolicyID,
long dsDelta, short oldRep, short newRep) {
EnumCounters<StorageType> typeSpaceDeltas =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5bb88c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 3b14eec..0459840 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -186,7 +186,6 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -207,7 +206,6 @@ import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretMan
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager.SecretManagerState;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockUnderConstructionFeature;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
@@ -3292,40 +3290,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final Block commitBlock) throws IOException {
assert hasWriteLock();
Preconditions.checkArgument(fileINode.isUnderConstruction());
- if (!blockManager.commitOrCompleteLastBlock(fileINode, commitBlock)) {
- return;
- }
-
- // Adjust disk space consumption if required
- final long diff;
- final short replicationFactor;
- if (fileINode.isStriped()) {
- final ErasureCodingPolicy ecPolicy = FSDirErasureCodingOp
- .getErasureCodingPolicy(this, iip);
- final short numDataUnits = (short) ecPolicy.getNumDataUnits();
- final short numParityUnits = (short) ecPolicy.getNumParityUnits();
-
- final long numBlocks = numDataUnits + numParityUnits;
- final long fullBlockGroupSize =
- fileINode.getPreferredBlockSize() * numBlocks;
-
- final BlockInfoStriped striped = new BlockInfoStriped(commitBlock,
- ecPolicy);
- final long actualBlockGroupSize = striped.spaceConsumed();
-
- diff = fullBlockGroupSize - actualBlockGroupSize;
- replicationFactor = (short) 1;
- } else {
- diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
- replicationFactor = fileINode.getFileReplication();
- }
- if (diff > 0) {
- try {
- dir.updateSpaceConsumed(iip, 0, -diff, replicationFactor);
- } catch (IOException e) {
- LOG.warn("Unexpected exception while updating disk space.", e);
- }
- }
+ blockManager.commitOrCompleteLastBlock(fileINode, commitBlock, iip);
}
void addCommittedBlocksToPending(final INodeFile pendingFile) {
@@ -3583,7 +3548,6 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* @param pendingFile open file that needs to be closed
* @param storedBlock last block
- * @return Path of the file that was closed.
* @throws IOException on error
*/
@VisibleForTesting
@@ -5734,6 +5698,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
/** @return the FSDirectory. */
+ @Override
public FSDirectory getFSDirectory() {
return dir;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5bb88c8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
index c675144..e07376b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Namesystem.java
@@ -30,6 +30,8 @@ public interface Namesystem extends RwLock, SafeMode {
BlockCollection getBlockCollection(long id);
+ FSDirectory getFSDirectory();
+
void startSecretManagerIfNecessary();
boolean isInSnapshot(long blockCollectionID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a5bb88c8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
index d459db0..deb5208 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
@@ -18,10 +18,14 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+import java.util.ArrayList;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.ContentSummary;
@@ -33,61 +37,79 @@ import org.apache.hadoop.hdfs.DFSOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.TestFileCreation;
import org.apache.hadoop.hdfs.client.HdfsDataOutputStream;
import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
+import org.apache.hadoop.hdfs.server.datanode.InternalDataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.ipc.RemoteException;
-import org.junit.After;
+import org.apache.hadoop.hdfs.protocolPB.DatanodeProtocolClientSideTranslatorPB;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.test.GenericTestUtils.LogCapturer;
+import org.junit.AfterClass;
import org.junit.Assert;
-import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestDiskspaceQuotaUpdate {
private static final int BLOCKSIZE = 1024;
private static final short REPLICATION = 4;
static final long seed = 0L;
- private static final Path dir = new Path("/TestQuotaUpdate");
+ private static final Path BASE_DIR = new Path("/TestQuotaUpdate");
- private Configuration conf;
- private MiniDFSCluster cluster;
- private FSDirectory fsdir;
- private DistributedFileSystem dfs;
+ private static Configuration conf;
+ private static MiniDFSCluster cluster;
- @Before
- public void setUp() throws Exception {
+ @BeforeClass
+ public static void setUp() throws Exception {
conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(REPLICATION)
.build();
cluster.waitActive();
-
- fsdir = cluster.getNamesystem().getFSDirectory();
- dfs = cluster.getFileSystem();
}
- @After
- public void tearDown() throws Exception {
+ @AfterClass
+ public static void tearDown() throws Exception {
if (cluster != null) {
cluster.shutdown();
cluster = null;
}
}
+ private Path getParent(String testName) {
+ return new Path(BASE_DIR, testName);
+ }
+
+ private FSDirectory getFSDirectory() {
+ return cluster.getNamesystem().getFSDirectory();
+ }
+
+ private DistributedFileSystem getDFS() throws IOException {
+ return cluster.getFileSystem();
+ }
+
/**
* Test if the quota can be correctly updated for create file
*/
@Test (timeout=60000)
public void testQuotaUpdateWithFileCreate() throws Exception {
- final Path foo = new Path(dir, "foo");
+ final Path foo =
+ new Path(getParent(GenericTestUtils.getMethodName()), "foo");
Path createdFile = new Path(foo, "created_file.data");
- dfs.mkdirs(foo);
- dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1);
+ getDFS().mkdirs(foo);
+ getDFS().setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1);
long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2;
- DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16,
+ DFSTestUtil.createFile(getDFS(), createdFile, BLOCKSIZE / 16,
fileLen, BLOCKSIZE, REPLICATION, seed);
- INode fnode = fsdir.getINode4Write(foo.toString());
+ INode fnode = getFSDirectory().getINode4Write(foo.toString());
assertTrue(fnode.isDirectory());
assertTrue(fnode.isQuotaSet());
QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
@@ -101,18 +123,20 @@ public class TestDiskspaceQuotaUpdate {
*/
@Test (timeout=60000)
public void testUpdateQuotaForAppend() throws Exception {
- final Path foo = new Path(dir ,"foo");
+ final Path foo =
+ new Path(getParent(GenericTestUtils.getMethodName()), "foo");
final Path bar = new Path(foo, "bar");
long currentFileLen = BLOCKSIZE;
- DFSTestUtil.createFile(dfs, bar, currentFileLen, REPLICATION, seed);
- dfs.setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1);
+ DFSTestUtil.createFile(getDFS(), bar, currentFileLen, REPLICATION, seed);
+ getDFS().setQuota(foo, Long.MAX_VALUE-1, Long.MAX_VALUE-1);
// append half of the block data, the previous file length is at block
// boundary
- DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE / 2);
+ DFSTestUtil.appendFile(getDFS(), bar, BLOCKSIZE / 2);
currentFileLen += (BLOCKSIZE / 2);
- INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
+ INodeDirectory fooNode =
+ getFSDirectory().getINode4Write(foo.toString()).asDirectory();
assertTrue(fooNode.isQuotaSet());
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed();
@@ -120,11 +144,11 @@ public class TestDiskspaceQuotaUpdate {
long ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
- ContentSummary c = dfs.getContentSummary(foo);
+ ContentSummary c = getDFS().getContentSummary(foo);
assertEquals(c.getSpaceConsumed(), ds);
// append another block, the previous file length is not at block boundary
- DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);
+ DFSTestUtil.appendFile(getDFS(), bar, BLOCKSIZE);
currentFileLen += BLOCKSIZE;
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
@@ -132,11 +156,11 @@ public class TestDiskspaceQuotaUpdate {
ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
- c = dfs.getContentSummary(foo);
+ c = getDFS().getContentSummary(foo);
assertEquals(c.getSpaceConsumed(), ds);
// append several blocks
- DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE * 3 + BLOCKSIZE / 8);
+ DFSTestUtil.appendFile(getDFS(), bar, BLOCKSIZE * 3 + BLOCKSIZE / 8);
currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8);
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
@@ -144,7 +168,7 @@ public class TestDiskspaceQuotaUpdate {
ds = quota.getStorageSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
- c = dfs.getContentSummary(foo);
+ c = getDFS().getContentSummary(foo);
assertEquals(c.getSpaceConsumed(), ds);
}
@@ -154,16 +178,18 @@ public class TestDiskspaceQuotaUpdate {
*/
@Test (timeout=60000)
public void testUpdateQuotaForFSync() throws Exception {
- final Path foo = new Path("/foo");
+ final Path foo =
+ new Path(getParent(GenericTestUtils.getMethodName()), "foo");
final Path bar = new Path(foo, "bar");
- DFSTestUtil.createFile(dfs, bar, BLOCKSIZE, REPLICATION, 0L);
- dfs.setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+ DFSTestUtil.createFile(getDFS(), bar, BLOCKSIZE, REPLICATION, 0L);
+ getDFS().setQuota(foo, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
- FSDataOutputStream out = dfs.append(bar);
+ FSDataOutputStream out = getDFS().append(bar);
out.write(new byte[BLOCKSIZE / 4]);
((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
- INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
+ INodeDirectory fooNode =
+ getFSDirectory().getINode4Write(foo.toString()).asDirectory();
QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed();
long ns = quota.getNameSpace();
@@ -174,7 +200,7 @@ public class TestDiskspaceQuotaUpdate {
out.write(new byte[BLOCKSIZE / 4]);
out.close();
- fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
+ fooNode = getFSDirectory().getINode4Write(foo.toString()).asDirectory();
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.getNameSpace();
ds = quota.getStorageSpace();
@@ -182,7 +208,7 @@ public class TestDiskspaceQuotaUpdate {
assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);
// append another block
- DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);
+ DFSTestUtil.appendFile(getDFS(), bar, BLOCKSIZE);
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
ns = quota.getNameSpace();
@@ -196,21 +222,21 @@ public class TestDiskspaceQuotaUpdate {
*/
@Test (timeout=60000)
public void testAppendOverStorageQuota() throws Exception {
- final Path dir = new Path("/TestAppendOverQuota");
+ final Path dir = getParent(GenericTestUtils.getMethodName());
final Path file = new Path(dir, "file");
// create partial block file
- dfs.mkdirs(dir);
- DFSTestUtil.createFile(dfs, file, BLOCKSIZE/2, REPLICATION, seed);
+ getDFS().mkdirs(dir);
+ DFSTestUtil.createFile(getDFS(), file, BLOCKSIZE/2, REPLICATION, seed);
// lower quota to cause exception when appending to partial block
- dfs.setQuota(dir, Long.MAX_VALUE - 1, 1);
- final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString())
- .asDirectory();
+ getDFS().setQuota(dir, Long.MAX_VALUE - 1, 1);
+ final INodeDirectory dirNode =
+ getFSDirectory().getINode4Write(dir.toString()).asDirectory();
final long spaceUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getStorageSpace();
try {
- DFSTestUtil.appendFile(dfs, file, BLOCKSIZE);
+ DFSTestUtil.appendFile(getDFS(), file, BLOCKSIZE);
Assert.fail("append didn't fail");
} catch (DSQuotaExceededException e) {
// ignore
@@ -218,7 +244,7 @@ public class TestDiskspaceQuotaUpdate {
LeaseManager lm = cluster.getNamesystem().getLeaseManager();
// check that the file exists, isn't UC, and has no dangling lease
- INodeFile inode = fsdir.getINode(file.toString()).asFile();
+ INodeFile inode = getFSDirectory().getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
Assert.assertNull("should not have a lease", lm.getLease(inode));
@@ -227,7 +253,7 @@ public class TestDiskspaceQuotaUpdate {
.getSpaceConsumed().getStorageSpace();
assertEquals(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
- dfs.recoverLease(file);
+ getDFS().recoverLease(file);
cluster.restartNameNodes();
}
@@ -237,23 +263,23 @@ public class TestDiskspaceQuotaUpdate {
*/
@Test (timeout=60000)
public void testAppendOverTypeQuota() throws Exception {
- final Path dir = new Path("/TestAppendOverTypeQuota");
+ final Path dir = getParent(GenericTestUtils.getMethodName());
final Path file = new Path(dir, "file");
// create partial block file
- dfs.mkdirs(dir);
+ getDFS().mkdirs(dir);
// set the storage policy on dir
- dfs.setStoragePolicy(dir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
- DFSTestUtil.createFile(dfs, file, BLOCKSIZE/2, REPLICATION, seed);
+ getDFS().setStoragePolicy(dir, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ DFSTestUtil.createFile(getDFS(), file, BLOCKSIZE/2, REPLICATION, seed);
// set quota of SSD to 1L
- dfs.setQuotaByStorageType(dir, StorageType.SSD, 1L);
- final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString())
- .asDirectory();
+ getDFS().setQuotaByStorageType(dir, StorageType.SSD, 1L);
+ final INodeDirectory dirNode =
+ getFSDirectory().getINode4Write(dir.toString()).asDirectory();
final long spaceUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getStorageSpace();
try {
- DFSTestUtil.appendFile(dfs, file, BLOCKSIZE);
+ DFSTestUtil.appendFile(getDFS(), file, BLOCKSIZE);
Assert.fail("append didn't fail");
} catch (QuotaByStorageTypeExceededException e) {
//ignore
@@ -261,7 +287,7 @@ public class TestDiskspaceQuotaUpdate {
// check that the file exists, isn't UC, and has no dangling lease
LeaseManager lm = cluster.getNamesystem().getLeaseManager();
- INodeFile inode = fsdir.getINode(file.toString()).asFile();
+ INodeFile inode = getFSDirectory().getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
Assert.assertNull("should not have a lease", lm.getLease(inode));
@@ -270,7 +296,7 @@ public class TestDiskspaceQuotaUpdate {
.getSpaceConsumed().getStorageSpace();
assertEquals(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
- dfs.recoverLease(file);
+ getDFS().recoverLease(file);
cluster.restartNameNodes();
}
@@ -279,21 +305,21 @@ public class TestDiskspaceQuotaUpdate {
*/
@Test (timeout=60000)
public void testTruncateOverQuota() throws Exception {
- final Path dir = new Path("/TestTruncateOverquota");
+ final Path dir = getParent(GenericTestUtils.getMethodName());
final Path file = new Path(dir, "file");
// create partial block file
- dfs.mkdirs(dir);
- DFSTestUtil.createFile(dfs, file, BLOCKSIZE/2, REPLICATION, seed);
+ getDFS().mkdirs(dir);
+ DFSTestUtil.createFile(getDFS(), file, BLOCKSIZE/2, REPLICATION, seed);
// lower quota to cause exception when appending to partial block
- dfs.setQuota(dir, Long.MAX_VALUE - 1, 1);
- final INodeDirectory dirNode = fsdir.getINode4Write(dir.toString())
- .asDirectory();
+ getDFS().setQuota(dir, Long.MAX_VALUE - 1, 1);
+ final INodeDirectory dirNode =
+ getFSDirectory().getINode4Write(dir.toString()).asDirectory();
final long spaceUsed = dirNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed().getStorageSpace();
try {
- dfs.truncate(file, BLOCKSIZE / 2 - 1);
+ getDFS().truncate(file, BLOCKSIZE / 2 - 1);
Assert.fail("truncate didn't fail");
} catch (RemoteException e) {
assertTrue(e.getClassName().contains("DSQuotaExceededException"));
@@ -301,7 +327,7 @@ public class TestDiskspaceQuotaUpdate {
// check that the file exists, isn't UC, and has no dangling lease
LeaseManager lm = cluster.getNamesystem().getLeaseManager();
- INodeFile inode = fsdir.getINode(file.toString()).asFile();
+ INodeFile inode = getFSDirectory().getINode(file.toString()).asFile();
Assert.assertNotNull(inode);
Assert.assertFalse("should not be UC", inode.isUnderConstruction());
Assert.assertNull("should not have a lease", lm.getLease(inode));
@@ -310,7 +336,7 @@ public class TestDiskspaceQuotaUpdate {
.getSpaceConsumed().getStorageSpace();
assertEquals(spaceUsed, newSpaceUsed);
// make sure edits aren't corrupted
- dfs.recoverLease(file);
+ getDFS().recoverLease(file);
cluster.restartNameNodes();
}
@@ -320,33 +346,34 @@ public class TestDiskspaceQuotaUpdate {
@Test
public void testQuotaInitialization() throws Exception {
final int size = 500;
- Path testDir = new Path("/testDir");
+ Path testDir =
+ new Path(getParent(GenericTestUtils.getMethodName()), "testDir");
long expectedSize = 3 * BLOCKSIZE + BLOCKSIZE/2;
- dfs.mkdirs(testDir);
- dfs.setQuota(testDir, size*4, expectedSize*size*2);
+ getDFS().mkdirs(testDir);
+ getDFS().setQuota(testDir, size*4, expectedSize*size*2);
Path[] testDirs = new Path[size];
for (int i = 0; i < size; i++) {
testDirs[i] = new Path(testDir, "sub" + i);
- dfs.mkdirs(testDirs[i]);
- dfs.setQuota(testDirs[i], 100, 1000000);
- DFSTestUtil.createFile(dfs, new Path(testDirs[i], "a"), expectedSize,
+ getDFS().mkdirs(testDirs[i]);
+ getDFS().setQuota(testDirs[i], 100, 1000000);
+ DFSTestUtil.createFile(getDFS(), new Path(testDirs[i], "a"), expectedSize,
(short)1, 1L);
}
// Directly access the name system to obtain the current cached usage.
- INodeDirectory root = fsdir.getRoot();
+ INodeDirectory root = getFSDirectory().getRoot();
HashMap<String, Long> nsMap = new HashMap<String, Long>();
HashMap<String, Long> dsMap = new HashMap<String, Long>();
scanDirsWithQuota(root, nsMap, dsMap, false);
- fsdir.updateCountForQuota(1);
+ getFSDirectory().updateCountForQuota(1);
scanDirsWithQuota(root, nsMap, dsMap, true);
- fsdir.updateCountForQuota(2);
+ getFSDirectory().updateCountForQuota(2);
scanDirsWithQuota(root, nsMap, dsMap, true);
- fsdir.updateCountForQuota(4);
+ getFSDirectory().updateCountForQuota(4);
scanDirsWithQuota(root, nsMap, dsMap, true);
}
@@ -372,4 +399,130 @@ public class TestDiskspaceQuotaUpdate {
}
}
}
+
+ /**
+ * Test that the cached quota stays correct between the COMMIT
+ * and COMPLETE block steps, even if the replication factor is
+ * changed during this time.
+ */
+ @Test (timeout=60000)
+ public void testQuotaIssuesWhileCommitting() throws Exception {
+ // We want a one-DN cluster so that we can force a lack of
+ // commit by only instrumenting a single DN; we kill the other 3
+ List<MiniDFSCluster.DataNodeProperties> dnprops = new ArrayList<>();
+ try {
+ for (int i = REPLICATION - 1; i > 0; i--) {
+ dnprops.add(cluster.stopDataNode(i));
+ }
+
+ DatanodeProtocolClientSideTranslatorPB nnSpy =
+ InternalDataNodeTestUtils.spyOnBposToNN(
+ cluster.getDataNodes().get(0), cluster.getNameNode());
+
+ testQuotaIssuesWhileCommittingHelper(nnSpy, (short) 1, (short) 4);
+ testQuotaIssuesWhileCommittingHelper(nnSpy, (short) 4, (short) 1);
+
+ // Don't actually change replication; just check that the sizes
+ // agree during the commit period
+ testQuotaIssuesWhileCommittingHelper(nnSpy, (short) 1, (short) 1);
+ } finally {
+ for (MiniDFSCluster.DataNodeProperties dnprop : dnprops) {
+ cluster.restartDataNode(dnprop, true);
+ }
+ cluster.waitActive();
+ }
+ }
+
+ private void testQuotaIssuesWhileCommittingHelper(
+ DatanodeProtocolClientSideTranslatorPB nnSpy,
+ final short initialReplication, final short finalReplication)
+ throws Exception {
+ final String logStmt =
+ "BUG: Inconsistent storagespace for directory";
+ final Path dir = new Path(getParent(GenericTestUtils.getMethodName()),
+ String.format("%d-%d", initialReplication, finalReplication));
+ final Path file = new Path(dir, "testfile");
+
+ LogCapturer logs = LogCapturer.captureLogs(NameNode.LOG);
+
+ Mockito.doAnswer(new Answer<Object>() {
+ @Override
+ public Object answer(InvocationOnMock invocation) throws Throwable {
+ if (finalReplication != initialReplication) {
+ getDFS().setReplication(file, finalReplication);
+ }
+ // Call getContentSummary before the DN can notify the NN
+ // that the block has been received to check for discrepancy
+ getDFS().getContentSummary(dir);
+ invocation.callRealMethod();
+ return null;
+ }
+ }).when(nnSpy).blockReceivedAndDeleted(
+ Mockito.<DatanodeRegistration>anyObject(),
+ Mockito.anyString(),
+ Mockito.<StorageReceivedDeletedBlocks[]>anyObject()
+ );
+
+ getDFS().mkdirs(dir);
+ getDFS().setQuota(dir, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+
+ DFSTestUtil.createFile(getDFS(), file, BLOCKSIZE/2, initialReplication, 1L);
+
+ // Also check for discrepancy after completing the file
+ getDFS().getContentSummary(dir);
+ assertFalse(logs.getOutput().contains(logStmt));
+ }
+
+ /**
+ * Test that the cached quota remains correct when the block has been
+ * written to but not yet committed, even if the replication factor
+ * is updated during this time.
+ */
+ private void testQuotaIssuesBeforeCommitting(short initialReplication,
+ short finalReplication) throws Exception {
+ final String logStmt =
+ "BUG: Inconsistent storagespace for directory";
+ final Path dir = new Path(getParent(GenericTestUtils.getMethodName()),
+ String.format("%d-%d", initialReplication, finalReplication));
+ final Path file = new Path(dir, "testfile");
+
+ LogCapturer logs = LogCapturer.captureLogs(NameNode.LOG);
+
+ getDFS().mkdirs(dir);
+ getDFS().setQuota(dir, Long.MAX_VALUE - 1, Long.MAX_VALUE - 1);
+
+ FSDataOutputStream out =
+ TestFileCreation.createFile(getDFS(), file, initialReplication);
+ TestFileCreation.writeFile(out, BLOCKSIZE / 2);
+ out.hflush();
+
+ getDFS().getContentSummary(dir);
+ if (finalReplication != initialReplication) {
+ // While the block is visible to the NN but has not yet been committed,
+ // change the replication
+ getDFS().setReplication(file, finalReplication);
+ }
+
+ out.close();
+
+ getDFS().getContentSummary(dir);
+ assertFalse(logs.getOutput().contains(logStmt));
+ }
+
+ @Test (timeout=60000)
+ public void testCachedComputedSizesAgreeBeforeCommitting() throws Exception {
+ // Don't actually change replication; just check that the sizes
+ // agree before the commit period
+ testQuotaIssuesBeforeCommitting((short)1, (short)1);
+ }
+
+ @Test (timeout=60000)
+ public void testDecreaseReplicationBeforeCommitting() throws Exception {
+ testQuotaIssuesBeforeCommitting((short)4, (short)1);
+ }
+
+ @Test (timeout=60000)
+ public void testIncreaseReplicationBeforeCommitting() throws Exception {
+ testQuotaIssuesBeforeCommitting((short)1, (short)4);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-5657. Fix
TestDefaultContainerExecutor. (asuresh)
Posted by ji...@apache.org.
YARN-5657. Fix TestDefaultContainerExecutor. (asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f67237cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f67237cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f67237cb
Branch: refs/heads/MAPREDUCE-6608
Commit: f67237cbe7bc48a1b9088e990800b37529f1db2a
Parents: 4174b97
Author: Arun Suresh <as...@apache.org>
Authored: Sat Sep 17 09:29:03 2016 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Sat Sep 17 09:32:05 2016 -0700
----------------------------------------------------------------------
.../server/nodemanager/TestDefaultContainerExecutor.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f67237cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
index e2ec693..3bb8008 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
@@ -22,6 +22,7 @@ import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.mock;
@@ -386,8 +387,8 @@ public class TestDefaultContainerExecutor {
}
}
return null;
- }
- }).when(mockUtil).copy(any(Path.class), any(Path.class));
+ }}).when(mockUtil).copy(any(Path.class), any(Path.class),
+ anyBoolean(), anyBoolean());
doAnswer(new Answer() {
@Override
@@ -478,7 +479,8 @@ public class TestDefaultContainerExecutor {
}
// Verify that the calls happen the expected number of times
- verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class));
+ verify(mockUtil, times(1)).copy(any(Path.class), any(Path.class),
+ anyBoolean(), anyBoolean());
verify(mockLfs, times(2)).getFsStatus(any(Path.class));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-10866. Fix Eclipse Java
compile errors related to generic parameters. Contributed by Konstantin
Shvachko.
Posted by ji...@apache.org.
HDFS-10866. Fix Eclipse Java compile errors related to generic parameters. Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6eb700ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6eb700ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6eb700ee
Branch: refs/heads/MAPREDUCE-6608
Commit: 6eb700ee7469cd77524676d62cbd20043f111288
Parents: 74b3dd5
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Fri Sep 23 11:25:15 2016 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Sep 23 14:10:11 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/io/MapFile.java | 4 +--
.../hadoop/fs/TestDelegationTokenRenewer.java | 3 +--
.../hadoop/hdfs/web/TestWebHdfsTokens.java | 27 +++++++++-----------
3 files changed, 15 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eb700ee/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
index ee76458..96a4189 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MapFile.java
@@ -990,8 +990,8 @@ public class MapFile {
reader.getKeyClass().asSubclass(WritableComparable.class),
reader.getValueClass());
- WritableComparable key = ReflectionUtils.newInstance(reader.getKeyClass()
- .asSubclass(WritableComparable.class), conf);
+ WritableComparable<?> key = ReflectionUtils.newInstance(
+ reader.getKeyClass().asSubclass(WritableComparable.class), conf);
Writable value = ReflectionUtils.newInstance(reader.getValueClass()
.asSubclass(Writable.class), conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eb700ee/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
index 6b66222..582bc31 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestDelegationTokenRenewer.java
@@ -49,7 +49,6 @@ public class TestDelegationTokenRenewer {
renewer = DelegationTokenRenewer.getInstance();
}
- @SuppressWarnings("unchecked")
@Test
public void testAddRemoveRenewAction() throws IOException,
InterruptedException {
@@ -81,7 +80,7 @@ public class TestDelegationTokenRenewer {
verify(token).cancel(eq(conf));
verify(fs, never()).getDelegationToken(null);
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
assertEquals("FileSystem not removed from DelegationTokenRenewer", 0,
renewer.getRenewQueueLength());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6eb700ee/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
index 6192ad9..058f63f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsTokens.java
@@ -127,7 +127,7 @@ public class TestWebHdfsTokens {
fs.toUrl(op, null);
verify(fs, never()).getDelegationToken();
verify(fs, never()).getDelegationToken(null);
- verify(fs, never()).setDelegationToken((Token<?>)any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
}
@Test(timeout = 1000)
@@ -160,7 +160,6 @@ public class TestWebHdfsTokens {
}
}
- @SuppressWarnings("unchecked") // for any(Token.class)
@Test
public void testLazyTokenFetchForWebhdfs() throws Exception {
MiniDFSCluster cluster = null;
@@ -190,7 +189,6 @@ public class TestWebHdfsTokens {
}
}
- @SuppressWarnings("unchecked") // for any(Token.class)
@Test
public void testLazyTokenFetchForSWebhdfs() throws Exception {
MiniDFSCluster cluster = null;
@@ -298,7 +296,6 @@ public class TestWebHdfsTokens {
}
}
- @SuppressWarnings("unchecked")
private void validateLazyTokenFetch(final Configuration clusterConf) throws Exception{
final String testUser = "DummyUser";
UserGroupInformation ugi = UserGroupInformation.createUserForTesting(
@@ -316,7 +313,7 @@ public class TestWebHdfsTokens {
fs.cancelDelegationToken(token);
verify(fs, never()).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
Assert.assertNull(fs.getRenewToken());
reset(fs);
@@ -326,7 +323,7 @@ public class TestWebHdfsTokens {
verify(fs, times(1)).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, times(1)).getDelegationToken(anyString());
- verify(fs, times(1)).setDelegationToken(any(Token.class));
+ verify(fs, times(1)).setDelegationToken(any());
token = fs.getRenewToken();
Assert.assertNotNull(token);
Assert.assertEquals(testUser, getTokenOwner(token));
@@ -338,7 +335,7 @@ public class TestWebHdfsTokens {
verify(fs, times(1)).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
Token<?> token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertEquals(fs.getTokenKind(), token.getKind());
@@ -358,7 +355,7 @@ public class TestWebHdfsTokens {
verify(fs, never()).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertEquals(fs.getTokenKind(), token.getKind());
@@ -376,7 +373,7 @@ public class TestWebHdfsTokens {
verify(fs, never()).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertEquals(fs.getTokenKind(), token.getKind());
@@ -390,7 +387,7 @@ public class TestWebHdfsTokens {
verify(fs, times(2)).getDelegationToken(); // first bad, then good
verify(fs, times(1)).replaceExpiredDelegationToken();
verify(fs, times(1)).getDelegationToken(null);
- verify(fs, times(1)).setDelegationToken(any(Token.class));
+ verify(fs, times(1)).setDelegationToken(any());
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertNotSame(token, token2);
@@ -407,7 +404,7 @@ public class TestWebHdfsTokens {
verify(fs, times(2)).getDelegationToken(); // first bad, then good
verify(fs, times(1)).replaceExpiredDelegationToken();
verify(fs, times(1)).getDelegationToken(null);
- verify(fs, times(1)).setDelegationToken(any(Token.class));
+ verify(fs, times(1)).setDelegationToken(any());
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertNotSame(token, token2);
@@ -420,7 +417,7 @@ public class TestWebHdfsTokens {
verify(fs, never()).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
verify(fs, times(1)).cancelDelegationToken(eq(token2));
// add a token to ugi for a new fs, verify it uses that token
@@ -449,7 +446,7 @@ public class TestWebHdfsTokens {
verify(fs, times(1)).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertEquals(fs.getTokenKind(), token.getKind());
@@ -469,7 +466,7 @@ public class TestWebHdfsTokens {
verify(fs, times(1)).getDelegationToken();
verify(fs, times(1)).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
token2 = fs.getRenewToken();
Assert.assertNotNull(token2);
Assert.assertEquals(fs.getTokenKind(), token.getKind());
@@ -482,7 +479,7 @@ public class TestWebHdfsTokens {
verify(fs, never()).getDelegationToken();
verify(fs, never()).replaceExpiredDelegationToken();
verify(fs, never()).getDelegationToken(anyString());
- verify(fs, never()).setDelegationToken(any(Token.class));
+ verify(fs, never()).setDelegationToken(any());
verify(fs, never()).cancelDelegationToken(any(Token.class));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-3141. Improve locks in
SchedulerApplicationAttempt/FSAppAttempt/FiCaSchedulerApp. Contributed by
Wangda Tan
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8a30f2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index 9e5a807..3555faa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -123,65 +123,72 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return queue.getMetrics();
}
- synchronized public void containerCompleted(RMContainer rmContainer,
+ public void containerCompleted(RMContainer rmContainer,
ContainerStatus containerStatus, RMContainerEventType event) {
-
- Container container = rmContainer.getContainer();
- ContainerId containerId = container.getId();
-
- // Remove from the list of newly allocated containers if found
- newlyAllocatedContainers.remove(rmContainer);
-
- // Inform the container
- rmContainer.handle(
- new RMContainerFinishedEvent(
- containerId,
- containerStatus,
- event)
- );
- if (LOG.isDebugEnabled()) {
- LOG.debug("Completed container: " + rmContainer.getContainerId() +
- " in state: " + rmContainer.getState() + " event:" + event);
- }
+ try {
+ writeLock.lock();
+ Container container = rmContainer.getContainer();
+ ContainerId containerId = container.getId();
+
+ // Remove from the list of newly allocated containers if found
+ newlyAllocatedContainers.remove(rmContainer);
+
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerFinishedEvent(containerId, containerStatus, event));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Completed container: " + rmContainer.getContainerId()
+ + " in state: " + rmContainer.getState() + " event:" + event);
+ }
+
+ // Remove from the list of containers
+ liveContainers.remove(rmContainer.getContainerId());
- // Remove from the list of containers
- liveContainers.remove(rmContainer.getContainerId());
+ Resource containerResource = rmContainer.getContainer().getResource();
+ RMAuditLogger.logSuccess(getUser(), AuditConstants.RELEASE_CONTAINER,
+ "SchedulerApp", getApplicationId(), containerId, containerResource);
- Resource containerResource = rmContainer.getContainer().getResource();
- RMAuditLogger.logSuccess(getUser(),
- AuditConstants.RELEASE_CONTAINER, "SchedulerApp",
- getApplicationId(), containerId, containerResource);
-
- // Update usage metrics
- queue.getMetrics().releaseResources(getUser(), 1, containerResource);
- this.attemptResourceUsage.decUsed(containerResource);
+ // Update usage metrics
+ queue.getMetrics().releaseResources(getUser(), 1, containerResource);
+ this.attemptResourceUsage.decUsed(containerResource);
- // remove from preemption map if it is completed
- preemptionMap.remove(rmContainer);
+ // remove from preemption map if it is completed
+ preemptionMap.remove(rmContainer);
- // Clear resource utilization metrics cache.
- lastMemoryAggregateAllocationUpdateTime = -1;
+ // Clear resource utilization metrics cache.
+ lastMemoryAggregateAllocationUpdateTime = -1;
+ } finally {
+ writeLock.unlock();
+ }
}
- private synchronized void unreserveInternal(
+ private void unreserveInternal(
SchedulerRequestKey schedulerKey, FSSchedulerNode node) {
- Map<NodeId, RMContainer> reservedContainers =
- this.reservedContainers.get(schedulerKey);
- RMContainer reservedContainer = reservedContainers.remove(node.getNodeID());
- if (reservedContainers.isEmpty()) {
- this.reservedContainers.remove(schedulerKey);
- }
-
- // Reset the re-reservation count
- resetReReservations(schedulerKey);
+ try {
+ writeLock.lock();
+ Map<NodeId, RMContainer> reservedContainers = this.reservedContainers.get(
+ schedulerKey);
+ RMContainer reservedContainer = reservedContainers.remove(
+ node.getNodeID());
+ if (reservedContainers.isEmpty()) {
+ this.reservedContainers.remove(schedulerKey);
+ }
+
+ // Reset the re-reservation count
+ resetReReservations(schedulerKey);
- Resource resource = reservedContainer.getContainer().getResource();
- this.attemptResourceUsage.decReserved(resource);
+ Resource resource = reservedContainer.getContainer().getResource();
+ this.attemptResourceUsage.decReserved(resource);
- LOG.info("Application " + getApplicationId() + " unreserved " + " on node "
- + node + ", currently has " + reservedContainers.size()
- + " at priority " + schedulerKey.getPriority() + "; currentReservation "
- + this.attemptResourceUsage.getReserved());
+ LOG.info(
+ "Application " + getApplicationId() + " unreserved " + " on node "
+ + node + ", currently has " + reservedContainers.size()
+ + " at priority " + schedulerKey.getPriority()
+ + "; currentReservation " + this.attemptResourceUsage
+ .getReserved());
+ } finally {
+ writeLock.unlock();
+ }
}
private void subtractResourcesOnBlacklistedNodes(
@@ -239,17 +246,6 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
return headroom;
}
- public synchronized float getLocalityWaitFactor(
- SchedulerRequestKey schedulerKey, int clusterNodes) {
- // Estimate: Required unique resources (i.e. hosts + racks)
- int requiredResources =
- Math.max(this.getResourceRequests(schedulerKey).size() - 1, 0);
-
- // waitFactor can't be more than '1'
- // i.e. no point skipping more than clustersize opportunities
- return Math.min(((float)requiredResources / clusterNodes), 1.0f);
- }
-
/**
* Return the level at which we are allowed to schedule containers, given the
* current size of the cluster and thresholds indicating how many nodes to
@@ -261,44 +257,56 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @param rackLocalityThreshold rackLocalityThreshold
* @return NodeType
*/
- public synchronized NodeType getAllowedLocalityLevel(
+ NodeType getAllowedLocalityLevel(
SchedulerRequestKey schedulerKey, int numNodes,
double nodeLocalityThreshold, double rackLocalityThreshold) {
// upper limit on threshold
- if (nodeLocalityThreshold > 1.0) { nodeLocalityThreshold = 1.0; }
- if (rackLocalityThreshold > 1.0) { rackLocalityThreshold = 1.0; }
+ if (nodeLocalityThreshold > 1.0) {
+ nodeLocalityThreshold = 1.0;
+ }
+ if (rackLocalityThreshold > 1.0) {
+ rackLocalityThreshold = 1.0;
+ }
// If delay scheduling is not being used, can schedule anywhere
if (nodeLocalityThreshold < 0.0 || rackLocalityThreshold < 0.0) {
return NodeType.OFF_SWITCH;
}
- // Default level is NODE_LOCAL
- if (!allowedLocalityLevel.containsKey(schedulerKey)) {
- allowedLocalityLevel.put(schedulerKey, NodeType.NODE_LOCAL);
- return NodeType.NODE_LOCAL;
- }
-
- NodeType allowed = allowedLocalityLevel.get(schedulerKey);
+ try {
+ writeLock.lock();
- // If level is already most liberal, we're done
- if (allowed.equals(NodeType.OFF_SWITCH)) return NodeType.OFF_SWITCH;
+ // Default level is NODE_LOCAL
+ if (!allowedLocalityLevel.containsKey(schedulerKey)) {
+ allowedLocalityLevel.put(schedulerKey, NodeType.NODE_LOCAL);
+ return NodeType.NODE_LOCAL;
+ }
- double threshold = allowed.equals(NodeType.NODE_LOCAL) ? nodeLocalityThreshold :
- rackLocalityThreshold;
+ NodeType allowed = allowedLocalityLevel.get(schedulerKey);
- // Relax locality constraints once we've surpassed threshold.
- if (getSchedulingOpportunities(schedulerKey) > (numNodes * threshold)) {
- if (allowed.equals(NodeType.NODE_LOCAL)) {
- allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
- resetSchedulingOpportunities(schedulerKey);
+ // If level is already most liberal, we're done
+ if (allowed.equals(NodeType.OFF_SWITCH)) {
+ return NodeType.OFF_SWITCH;
}
- else if (allowed.equals(NodeType.RACK_LOCAL)) {
- allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
- resetSchedulingOpportunities(schedulerKey);
+
+ double threshold = allowed.equals(NodeType.NODE_LOCAL) ?
+ nodeLocalityThreshold :
+ rackLocalityThreshold;
+
+ // Relax locality constraints once we've surpassed threshold.
+ if (getSchedulingOpportunities(schedulerKey) > (numNodes * threshold)) {
+ if (allowed.equals(NodeType.NODE_LOCAL)) {
+ allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
+ resetSchedulingOpportunities(schedulerKey);
+ } else if (allowed.equals(NodeType.RACK_LOCAL)) {
+ allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
+ resetSchedulingOpportunities(schedulerKey);
+ }
}
+ return allowedLocalityLevel.get(schedulerKey);
+ } finally {
+ writeLock.unlock();
}
- return allowedLocalityLevel.get(schedulerKey);
}
/**
@@ -311,119 +319,131 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @param currentTimeMs currentTimeMs
* @return NodeType
*/
- public synchronized NodeType getAllowedLocalityLevelByTime(
+ NodeType getAllowedLocalityLevelByTime(
SchedulerRequestKey schedulerKey, long nodeLocalityDelayMs,
long rackLocalityDelayMs, long currentTimeMs) {
-
// if not being used, can schedule anywhere
if (nodeLocalityDelayMs < 0 || rackLocalityDelayMs < 0) {
return NodeType.OFF_SWITCH;
}
- // default level is NODE_LOCAL
- if (!allowedLocalityLevel.containsKey(schedulerKey)) {
- // add the initial time of priority to prevent comparing with FsApp
- // startTime and allowedLocalityLevel degrade
- lastScheduledContainer.put(schedulerKey, currentTimeMs);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Init the lastScheduledContainer time, priority: "
- + schedulerKey.getPriority() + ", time: " + currentTimeMs);
+ try {
+ writeLock.lock();
+
+ // default level is NODE_LOCAL
+ if (!allowedLocalityLevel.containsKey(schedulerKey)) {
+ // add the initial time of priority to prevent comparing with FsApp
+ // startTime and allowedLocalityLevel degrade
+ lastScheduledContainer.put(schedulerKey, currentTimeMs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "Init the lastScheduledContainer time, priority: " + schedulerKey
+ .getPriority() + ", time: " + currentTimeMs);
+ }
+ allowedLocalityLevel.put(schedulerKey, NodeType.NODE_LOCAL);
+ return NodeType.NODE_LOCAL;
}
- allowedLocalityLevel.put(schedulerKey, NodeType.NODE_LOCAL);
- return NodeType.NODE_LOCAL;
- }
- NodeType allowed = allowedLocalityLevel.get(schedulerKey);
+ NodeType allowed = allowedLocalityLevel.get(schedulerKey);
- // if level is already most liberal, we're done
- if (allowed.equals(NodeType.OFF_SWITCH)) {
- return NodeType.OFF_SWITCH;
- }
-
- // check waiting time
- long waitTime = currentTimeMs;
- if (lastScheduledContainer.containsKey(schedulerKey)) {
- waitTime -= lastScheduledContainer.get(schedulerKey);
- } else {
- waitTime -= getStartTime();
- }
+ // if level is already most liberal, we're done
+ if (allowed.equals(NodeType.OFF_SWITCH)) {
+ return NodeType.OFF_SWITCH;
+ }
- long thresholdTime = allowed.equals(NodeType.NODE_LOCAL) ?
- nodeLocalityDelayMs : rackLocalityDelayMs;
+ // check waiting time
+ long waitTime = currentTimeMs;
+ if (lastScheduledContainer.containsKey(schedulerKey)) {
+ waitTime -= lastScheduledContainer.get(schedulerKey);
+ } else{
+ waitTime -= getStartTime();
+ }
- if (waitTime > thresholdTime) {
- if (allowed.equals(NodeType.NODE_LOCAL)) {
- allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
- resetSchedulingOpportunities(schedulerKey, currentTimeMs);
- } else if (allowed.equals(NodeType.RACK_LOCAL)) {
- allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
- resetSchedulingOpportunities(schedulerKey, currentTimeMs);
+ long thresholdTime = allowed.equals(NodeType.NODE_LOCAL) ?
+ nodeLocalityDelayMs :
+ rackLocalityDelayMs;
+
+ if (waitTime > thresholdTime) {
+ if (allowed.equals(NodeType.NODE_LOCAL)) {
+ allowedLocalityLevel.put(schedulerKey, NodeType.RACK_LOCAL);
+ resetSchedulingOpportunities(schedulerKey, currentTimeMs);
+ } else if (allowed.equals(NodeType.RACK_LOCAL)) {
+ allowedLocalityLevel.put(schedulerKey, NodeType.OFF_SWITCH);
+ resetSchedulingOpportunities(schedulerKey, currentTimeMs);
+ }
}
+ return allowedLocalityLevel.get(schedulerKey);
+ } finally {
+ writeLock.unlock();
}
- return allowedLocalityLevel.get(schedulerKey);
}
- synchronized public RMContainer allocate(NodeType type, FSSchedulerNode node,
+ public RMContainer allocate(NodeType type, FSSchedulerNode node,
SchedulerRequestKey schedulerKey, ResourceRequest request,
Container reservedContainer) {
- // Update allowed locality level
- NodeType allowed = allowedLocalityLevel.get(schedulerKey);
- if (allowed != null) {
- if (allowed.equals(NodeType.OFF_SWITCH) &&
- (type.equals(NodeType.NODE_LOCAL) ||
- type.equals(NodeType.RACK_LOCAL))) {
- this.resetAllowedLocalityLevel(schedulerKey, type);
+ RMContainer rmContainer;
+ Container container;
+
+ try {
+ writeLock.lock();
+ // Update allowed locality level
+ NodeType allowed = allowedLocalityLevel.get(schedulerKey);
+ if (allowed != null) {
+ if (allowed.equals(NodeType.OFF_SWITCH) && (type.equals(
+ NodeType.NODE_LOCAL) || type.equals(NodeType.RACK_LOCAL))) {
+ this.resetAllowedLocalityLevel(schedulerKey, type);
+ } else if (allowed.equals(NodeType.RACK_LOCAL) && type.equals(
+ NodeType.NODE_LOCAL)) {
+ this.resetAllowedLocalityLevel(schedulerKey, type);
+ }
}
- else if (allowed.equals(NodeType.RACK_LOCAL) &&
- type.equals(NodeType.NODE_LOCAL)) {
- this.resetAllowedLocalityLevel(schedulerKey, type);
+
+ // Required sanity check - AM can call 'allocate' to update resource
+ // request without locking the scheduler, hence we need to check
+ if (getTotalRequiredResources(schedulerKey) <= 0) {
+ return null;
}
- }
- // Required sanity check - AM can call 'allocate' to update resource
- // request without locking the scheduler, hence we need to check
- if (getTotalRequiredResources(schedulerKey) <= 0) {
- return null;
- }
+ container = reservedContainer;
+ if (container == null) {
+ container = createContainer(node, request.getCapability(),
+ schedulerKey);
+ }
- Container container = reservedContainer;
- if (container == null) {
- container =
- createContainer(node, request.getCapability(), schedulerKey);
- }
-
- // Create RMContainer
- RMContainer rmContainer = new RMContainerImpl(container,
- getApplicationAttemptId(), node.getNodeID(),
- appSchedulingInfo.getUser(), rmContext);
- ((RMContainerImpl)rmContainer).setQueueName(this.getQueueName());
+ // Create RMContainer
+ rmContainer = new RMContainerImpl(container,
+ getApplicationAttemptId(), node.getNodeID(),
+ appSchedulingInfo.getUser(), rmContext);
+ ((RMContainerImpl) rmContainer).setQueueName(this.getQueueName());
- // Add it to allContainers list.
- newlyAllocatedContainers.add(rmContainer);
- liveContainers.put(container.getId(), rmContainer);
+ // Add it to allContainers list.
+ newlyAllocatedContainers.add(rmContainer);
+ liveContainers.put(container.getId(), rmContainer);
- // Update consumption and track allocations
- List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
- type, node, schedulerKey, request, container);
- this.attemptResourceUsage.incUsed(container.getResource());
+ // Update consumption and track allocations
+ List<ResourceRequest> resourceRequestList = appSchedulingInfo.allocate(
+ type, node, schedulerKey, request, container);
+ this.attemptResourceUsage.incUsed(container.getResource());
- // Update resource requests related to "request" and store in RMContainer
- ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
+ // Update resource requests related to "request" and store in RMContainer
+ ((RMContainerImpl) rmContainer).setResourceRequests(resourceRequestList);
- // Inform the container
- rmContainer.handle(
- new RMContainerEvent(container.getId(), RMContainerEventType.START));
+ // Inform the container
+ rmContainer.handle(
+ new RMContainerEvent(container.getId(), RMContainerEventType.START));
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("allocate: applicationAttemptId=" + container.getId()
+ .getApplicationAttemptId() + " container=" + container.getId()
+ + " host=" + container.getNodeId().getHost() + " type=" + type);
+ }
+ RMAuditLogger.logSuccess(getUser(), AuditConstants.ALLOC_CONTAINER,
+ "SchedulerApp", getApplicationId(), container.getId(),
+ container.getResource());
+ } finally {
+ writeLock.unlock();
+ }
- if (LOG.isDebugEnabled()) {
- LOG.debug("allocate: applicationAttemptId="
- + container.getId().getApplicationAttemptId()
- + " container=" + container.getId() + " host="
- + container.getNodeId().getHost() + " type=" + type);
- }
- RMAuditLogger.logSuccess(getUser(),
- AuditConstants.ALLOC_CONTAINER, "SchedulerApp",
- getApplicationId(), container.getId(), container.getResource());
-
return rmContainer;
}
@@ -434,19 +454,30 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
* @param schedulerKey Scheduler Key
* @param level NodeType
*/
- public synchronized void resetAllowedLocalityLevel(
+ public void resetAllowedLocalityLevel(
SchedulerRequestKey schedulerKey, NodeType level) {
- NodeType old = allowedLocalityLevel.get(schedulerKey);
- LOG.info("Raising locality level from " + old + " to " + level + " at " +
- " priority " + schedulerKey.getPriority());
- allowedLocalityLevel.put(schedulerKey, level);
+ NodeType old;
+ try {
+ writeLock.lock();
+ old = allowedLocalityLevel.put(schedulerKey, level);
+ } finally {
+ writeLock.unlock();
+ }
+
+ LOG.info("Raising locality level from " + old + " to " + level + " at "
+ + " priority " + schedulerKey.getPriority());
}
// related methods
public void addPreemption(RMContainer container, long time) {
assert preemptionMap.get(container) == null;
- preemptionMap.put(container, time);
- Resources.addTo(preemptedResources, container.getAllocatedResource());
+ try {
+ writeLock.lock();
+ preemptionMap.put(container, time);
+ Resources.addTo(preemptedResources, container.getAllocatedResource());
+ } finally {
+ writeLock.unlock();
+ }
}
public Long getContainerPreemptionTime(RMContainer container) {
@@ -584,21 +615,35 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
getUser(), rmContainer.getContainer().getResource());
}
- private synchronized void setReservation(SchedulerNode node) {
- String rackName = node.getRackName() == null ? "NULL" : node.getRackName();
- Set<String> rackReservations = reservations.get(rackName);
- if (rackReservations == null) {
- rackReservations = new HashSet<>();
- reservations.put(rackName, rackReservations);
+ private void setReservation(SchedulerNode node) {
+ String rackName =
+ node.getRackName() == null ? "NULL" : node.getRackName();
+
+ try {
+ writeLock.lock();
+ Set<String> rackReservations = reservations.get(rackName);
+ if (rackReservations == null) {
+ rackReservations = new HashSet<>();
+ reservations.put(rackName, rackReservations);
+ }
+ rackReservations.add(node.getNodeName());
+ } finally {
+ writeLock.unlock();
}
- rackReservations.add(node.getNodeName());
}
- private synchronized void clearReservation(SchedulerNode node) {
- String rackName = node.getRackName() == null ? "NULL" : node.getRackName();
- Set<String> rackReservations = reservations.get(rackName);
- if (rackReservations != null) {
- rackReservations.remove(node.getNodeName());
+ private void clearReservation(SchedulerNode node) {
+ String rackName =
+ node.getRackName() == null ? "NULL" : node.getRackName();
+
+ try {
+ writeLock.lock();
+ Set<String> rackReservations = reservations.get(rackName);
+ if (rackReservations != null) {
+ rackReservations.remove(node.getNodeName());
+ }
+ } finally {
+ writeLock.unlock();
}
}
@@ -737,7 +782,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// For each priority, see if we can schedule a node local, rack local
// or off-switch request. Rack of off-switch requests may be delayed
// (not scheduled) in order to promote better locality.
- synchronized (this) {
+ try {
+ writeLock.lock();
for (SchedulerRequestKey schedulerKey : keysToTry) {
// Skip it for reserved container, since
// we already check it in isValidReservation.
@@ -772,8 +818,8 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
&& localRequest != null && localRequest.getNumContainers() != 0) {
- return assignContainer(node, localRequest,
- NodeType.NODE_LOCAL, reserved, schedulerKey);
+ return assignContainer(node, localRequest, NodeType.NODE_LOCAL,
+ reserved, schedulerKey);
}
if (rackLocalRequest != null && !rackLocalRequest.getRelaxLocality()) {
@@ -781,29 +827,31 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
}
if (rackLocalRequest != null && rackLocalRequest.getNumContainers() != 0
- && (allowedLocality.equals(NodeType.RACK_LOCAL) ||
- allowedLocality.equals(NodeType.OFF_SWITCH))) {
- return assignContainer(node, rackLocalRequest,
- NodeType.RACK_LOCAL, reserved, schedulerKey);
+ && (allowedLocality.equals(NodeType.RACK_LOCAL) || allowedLocality
+ .equals(NodeType.OFF_SWITCH))) {
+ return assignContainer(node, rackLocalRequest, NodeType.RACK_LOCAL,
+ reserved, schedulerKey);
}
- ResourceRequest offSwitchRequest =
- getResourceRequest(schedulerKey, ResourceRequest.ANY);
+ ResourceRequest offSwitchRequest = getResourceRequest(schedulerKey,
+ ResourceRequest.ANY);
if (offSwitchRequest != null && !offSwitchRequest.getRelaxLocality()) {
continue;
}
- if (offSwitchRequest != null &&
- offSwitchRequest.getNumContainers() != 0) {
- if (!hasNodeOrRackLocalRequests(schedulerKey) ||
- allowedLocality.equals(NodeType.OFF_SWITCH)) {
- return assignContainer(
- node, offSwitchRequest, NodeType.OFF_SWITCH, reserved,
- schedulerKey);
+ if (offSwitchRequest != null
+ && offSwitchRequest.getNumContainers() != 0) {
+ if (!hasNodeOrRackLocalRequests(schedulerKey) || allowedLocality
+ .equals(NodeType.OFF_SWITCH)) {
+ return assignContainer(node, offSwitchRequest, NodeType.OFF_SWITCH,
+ reserved, schedulerKey);
}
}
}
+ } finally {
+ writeLock.unlock();
}
+
return Resources.none();
}
@@ -963,14 +1011,17 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
Resources.addTo(demand, getCurrentConsumption());
// Add up outstanding resource requests
- synchronized (this) {
+ try {
+ writeLock.lock();
for (SchedulerRequestKey k : getSchedulerKeys()) {
ResourceRequest r = getResourceRequest(k, ResourceRequest.ANY);
if (r != null) {
- Resources.multiplyAndAddTo(demand,
- r.getCapability(), r.getNumContainers());
+ Resources.multiplyAndAddTo(demand, r.getCapability(),
+ r.getNumContainers());
}
}
+ } finally {
+ writeLock.unlock();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-4591. YARN Web UIs should
provide a robots.txt. (Sidharta Seethana via wangda)
Posted by ji...@apache.org.
YARN-4591. YARN Web UIs should provide a robots.txt. (Sidharta Seethana via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a58bfee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a58bfee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a58bfee
Branch: refs/heads/MAPREDUCE-6608
Commit: 5a58bfee30a662b1b556048504f66f9cf00d182a
Parents: 0e918df
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Sep 20 17:20:50 2016 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 20 17:20:50 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/webapp/Dispatcher.java | 9 +++++
.../org/apache/hadoop/yarn/webapp/WebApp.java | 4 +-
.../hadoop/yarn/webapp/view/RobotsTextPage.java | 39 ++++++++++++++++++++
.../apache/hadoop/yarn/webapp/TestWebApp.java | 26 +++++++++++++
4 files changed, 77 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a58bfee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
index 66dd21b..d519dbb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/Dispatcher.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.http.HtmlQuoting;
import org.apache.hadoop.yarn.webapp.Controller.RequestContext;
import org.apache.hadoop.yarn.webapp.Router.Dest;
import org.apache.hadoop.yarn.webapp.view.ErrorPage;
+import org.apache.hadoop.yarn.webapp.view.RobotsTextPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -117,6 +118,14 @@ public class Dispatcher extends HttpServlet {
}
Controller.RequestContext rc =
injector.getInstance(Controller.RequestContext.class);
+
+ //short-circuit robots.txt serving for all YARN webapps.
+ if (uri.equals(RobotsTextPage.ROBOTS_TXT_PATH)) {
+ rc.setStatus(HttpServletResponse.SC_FOUND);
+ render(RobotsTextPage.class);
+ return;
+ }
+
if (setCookieParams(rc, req) > 0) {
Cookie ec = rc.cookies().get(ERROR_COOKIE);
if (ec != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a58bfee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
index 2c21d1b..fe800f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApp.java
@@ -29,6 +29,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.yarn.webapp.view.RobotsTextPage;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -158,7 +159,8 @@ public abstract class WebApp extends ServletModule {
public void configureServlets() {
setup();
- serve("/", "/__stop").with(Dispatcher.class);
+ serve("/", "/__stop", RobotsTextPage.ROBOTS_TXT_PATH)
+ .with(Dispatcher.class);
for (String path : this.servePathSpecs) {
serve(path).with(Dispatcher.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a58bfee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/RobotsTextPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/RobotsTextPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/RobotsTextPage.java
new file mode 100644
index 0000000..b15d492
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/RobotsTextPage.java
@@ -0,0 +1,39 @@
+/*
+ * *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * /
+ */
+
+package org.apache.hadoop.yarn.webapp.view;
+
+/**
+ * Simple class that renders a robot.txt page that disallows crawling.
+ */
+
+public class RobotsTextPage extends TextPage {
+ public static final String ROBOTS_TXT = "robots.txt";
+ public static final String ROBOTS_TXT_PATH = "/" + ROBOTS_TXT;
+
+ static final String USER_AGENT_LINE = "User-agent: *";
+ static final String DISALLOW_LINE = "Disallow: /";
+
+ @Override
+ public void render() {
+ putWithoutEscapeHtml(USER_AGENT_LINE);
+ putWithoutEscapeHtml(DISALLOW_LINE);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a58bfee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
index acec205..deef855 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/webapp/TestWebApp.java
@@ -38,6 +38,7 @@ import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.yarn.MockApps;
import org.apache.hadoop.yarn.webapp.view.HtmlPage;
import org.apache.hadoop.yarn.webapp.view.JQueryUI;
+import org.apache.hadoop.yarn.webapp.view.RobotsTextPage;
import org.apache.hadoop.yarn.webapp.view.TextPage;
import org.junit.Test;
import org.slf4j.Logger;
@@ -260,6 +261,31 @@ public class TestWebApp {
}
}
+ @Test public void testRobotsText() throws Exception {
+ WebApp app =
+ WebApps.$for("test", TestWebApp.class, this, "ws").start(new WebApp() {
+ @Override
+ public void setup() {
+ bind(MyTestJAXBContextResolver.class);
+ bind(MyTestWebService.class);
+ }
+ });
+ String baseUrl = baseUrl(app);
+ try {
+ //using system line separator here since that is what
+ // TextView (via PrintWriter) seems to use.
+ String[] robotsTxtOutput = getContent(baseUrl +
+ RobotsTextPage.ROBOTS_TXT).trim().split(System.getProperty("line"
+ + ".separator"));
+
+ assertEquals(2, robotsTxtOutput.length);
+ assertEquals("User-agent: *", robotsTxtOutput[0]);
+ assertEquals("Disallow: /", robotsTxtOutput[1]);
+ } finally {
+ app.stop();
+ }
+ }
+
// This is to test the GuiceFilter should only be applied to webAppContext,
// not to logContext;
@Test public void testYARNWebAppContext() throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-3140. Improve locks in
AbstractCSQueue/LeafQueue/ParentQueue. Contributed by Wangda Tan
Posted by ji...@apache.org.
YARN-3140. Improve locks in AbstractCSQueue/LeafQueue/ParentQueue. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b66d9ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b66d9ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b66d9ec
Branch: refs/heads/MAPREDUCE-6608
Commit: 2b66d9ec5bdaec7e6b278926fbb6f222c4e3afaa
Parents: e52d6e7
Author: Jian He <ji...@apache.org>
Authored: Tue Sep 20 15:03:07 2016 +0800
Committer: Jian He <ji...@apache.org>
Committed: Tue Sep 20 15:03:31 2016 +0800
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 10 +
.../scheduler/capacity/AbstractCSQueue.java | 378 ++--
.../scheduler/capacity/LeafQueue.java | 1819 ++++++++++--------
.../scheduler/capacity/ParentQueue.java | 825 ++++----
.../scheduler/capacity/PlanQueue.java | 122 +-
.../scheduler/capacity/ReservationQueue.java | 67 +-
.../capacity/TestContainerResizing.java | 4 +-
7 files changed, 1787 insertions(+), 1438 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index a5c0f71..01b1da7 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -564,4 +564,14 @@
\u2002\u2002\u2002\u2002</Or>
\u2002\u2002\u2002\u2002<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
\u2002\u2002</Match>
+
+ <!-- Ignore VO_VOLATILE_INCREMENT, they will be protected by writeLock -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue$User" />
+\u2002\u2002\u2002\u2002<Or>
+ <Field name="pendingApplications" />
+ <Field name="activeApplications" />
+\u2002\u2002\u2002\u2002</Or>
+ <Bug pattern="VO_VOLATILE_INCREMENT" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 1d8f929..096f5ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -24,6 +24,7 @@ import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -60,25 +61,25 @@ import com.google.common.collect.Sets;
public abstract class AbstractCSQueue implements CSQueue {
private static final Log LOG = LogFactory.getLog(AbstractCSQueue.class);
- CSQueue parent;
+ volatile CSQueue parent;
final String queueName;
volatile int numContainers;
final Resource minimumAllocation;
volatile Resource maximumAllocation;
- QueueState state;
+ volatile QueueState state;
final CSQueueMetrics metrics;
protected final PrivilegedEntity queueEntity;
final ResourceCalculator resourceCalculator;
Set<String> accessibleLabels;
- RMNodeLabelsManager labelManager;
+ final RMNodeLabelsManager labelManager;
String defaultLabelExpression;
Map<AccessType, AccessControlList> acls =
new HashMap<AccessType, AccessControlList>();
volatile boolean reservationsContinueLooking;
- private boolean preemptionDisabled;
+ private volatile boolean preemptionDisabled;
// Track resource usage-by-label like used-resource/pending-resource, etc.
volatile ResourceUsage queueUsage;
@@ -94,6 +95,9 @@ public abstract class AbstractCSQueue implements CSQueue {
protected ActivitiesManager activitiesManager;
+ protected ReentrantReadWriteLock.ReadLock readLock;
+ protected ReentrantReadWriteLock.WriteLock writeLock;
+
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
this.labelManager = cs.getRMContext().getNodeLabelManager();
@@ -116,7 +120,11 @@ public abstract class AbstractCSQueue implements CSQueue {
queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath());
// initialize QueueCapacities
- queueCapacities = new QueueCapacities(parent == null);
+ queueCapacities = new QueueCapacities(parent == null);
+
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
}
protected void setupConfigurableCapacities() {
@@ -128,12 +136,12 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized float getCapacity() {
+ public float getCapacity() {
return queueCapacities.getCapacity();
}
@Override
- public synchronized float getAbsoluteCapacity() {
+ public float getAbsoluteCapacity() {
return queueCapacities.getAbsoluteCapacity();
}
@@ -167,7 +175,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized QueueState getState() {
+ public QueueState getState() {
return state;
}
@@ -187,13 +195,13 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized CSQueue getParent() {
+ public CSQueue getParent() {
return parent;
}
@Override
- public synchronized void setParent(CSQueue newParentQueue) {
- this.parent = (ParentQueue)newParentQueue;
+ public void setParent(CSQueue newParentQueue) {
+ this.parent = newParentQueue;
}
public Set<String> getAccessibleNodeLabels() {
@@ -221,18 +229,22 @@ public abstract class AbstractCSQueue implements CSQueue {
* Set maximum capacity - used only for testing.
* @param maximumCapacity new max capacity
*/
- synchronized void setMaxCapacity(float maximumCapacity) {
- // Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(),
- queueCapacities.getCapacity(), maximumCapacity);
- float absMaxCapacity =
- CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(),
- queueCapacities.getAbsoluteCapacity(),
- absMaxCapacity);
-
- queueCapacities.setMaximumCapacity(maximumCapacity);
- queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
+ void setMaxCapacity(float maximumCapacity) {
+ try {
+ writeLock.lock();
+ // Sanity check
+ CSQueueUtils.checkMaxCapacity(getQueueName(),
+ queueCapacities.getCapacity(), maximumCapacity);
+ float absMaxCapacity = CSQueueUtils.computeAbsoluteMaximumCapacity(
+ maximumCapacity, parent);
+ CSQueueUtils.checkAbsoluteCapacity(getQueueName(),
+ queueCapacities.getAbsoluteCapacity(), absMaxCapacity);
+
+ queueCapacities.setMaximumCapacity(maximumCapacity);
+ queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
@@ -240,70 +252,82 @@ public abstract class AbstractCSQueue implements CSQueue {
return defaultLabelExpression;
}
- synchronized void setupQueueConfigs(Resource clusterResource)
+ void setupQueueConfigs(Resource clusterResource)
throws IOException {
- // get labels
- this.accessibleLabels =
- csContext.getConfiguration().getAccessibleNodeLabels(getQueuePath());
- this.defaultLabelExpression = csContext.getConfiguration()
- .getDefaultNodeLabelExpression(getQueuePath());
-
- // inherit from parent if labels not set
- if (this.accessibleLabels == null && parent != null) {
- this.accessibleLabels = parent.getAccessibleNodeLabels();
- }
-
- // inherit from parent if labels not set
- if (this.defaultLabelExpression == null && parent != null
- && this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
- this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
- }
+ try {
+ writeLock.lock();
+ // get labels
+ this.accessibleLabels =
+ csContext.getConfiguration().getAccessibleNodeLabels(getQueuePath());
+ this.defaultLabelExpression =
+ csContext.getConfiguration().getDefaultNodeLabelExpression(
+ getQueuePath());
+
+ // inherit from parent if labels not set
+ if (this.accessibleLabels == null && parent != null) {
+ this.accessibleLabels = parent.getAccessibleNodeLabels();
+ }
- // After we setup labels, we can setup capacities
- setupConfigurableCapacities();
-
- this.maximumAllocation =
- csContext.getConfiguration().getMaximumAllocationPerQueue(
- getQueuePath());
-
- authorizer = YarnAuthorizationProvider.getInstance(csContext.getConf());
-
- this.state = csContext.getConfiguration().getState(getQueuePath());
- this.acls = csContext.getConfiguration().getAcls(getQueuePath());
+ // inherit from parent if labels not set
+ if (this.defaultLabelExpression == null && parent != null
+ && this.accessibleLabels.containsAll(
+ parent.getAccessibleNodeLabels())) {
+ this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
+ }
- // Update metrics
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
-
- // Check if labels of this queue is a subset of parent queue, only do this
- // when we not root
- if (parent != null && parent.getParent() != null) {
- if (parent.getAccessibleNodeLabels() != null
- && !parent.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
- // if parent isn't "*", child shouldn't be "*" too
- if (this.getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
- throw new IOException("Parent's accessible queue is not ANY(*), "
- + "but child's accessible queue is *");
- } else {
- Set<String> diff =
- Sets.difference(this.getAccessibleNodeLabels(),
- parent.getAccessibleNodeLabels());
- if (!diff.isEmpty()) {
- throw new IOException("Some labels of child queue is not a subset "
- + "of parent queue, these labels=["
- + StringUtils.join(diff, ",") + "]");
+ // After we setup labels, we can setup capacities
+ setupConfigurableCapacities();
+
+ this.maximumAllocation =
+ csContext.getConfiguration().getMaximumAllocationPerQueue(
+ getQueuePath());
+
+ authorizer = YarnAuthorizationProvider.getInstance(csContext.getConf());
+
+ this.state = csContext.getConfiguration().getState(getQueuePath());
+ this.acls = csContext.getConfiguration().getAcls(getQueuePath());
+
+ // Update metrics
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ minimumAllocation, this, labelManager, null);
+
+ // Check if labels of this queue is a subset of parent queue, only do this
+ // when we not root
+ if (parent != null && parent.getParent() != null) {
+ if (parent.getAccessibleNodeLabels() != null && !parent
+ .getAccessibleNodeLabels().contains(RMNodeLabelsManager.ANY)) {
+ // if parent isn't "*", child shouldn't be "*" too
+ if (this.getAccessibleNodeLabels().contains(
+ RMNodeLabelsManager.ANY)) {
+ throw new IOException("Parent's accessible queue is not ANY(*), "
+ + "but child's accessible queue is *");
+ } else{
+ Set<String> diff = Sets.difference(this.getAccessibleNodeLabels(),
+ parent.getAccessibleNodeLabels());
+ if (!diff.isEmpty()) {
+ throw new IOException(
+ "Some labels of child queue is not a subset "
+ + "of parent queue, these labels=[" + StringUtils
+ .join(diff, ",") + "]");
+ }
}
}
}
- }
- this.reservationsContinueLooking = csContext.getConfiguration()
- .getReservationContinueLook();
+ this.reservationsContinueLooking =
+ csContext.getConfiguration().getReservationContinueLook();
- this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this);
+ this.preemptionDisabled = isQueueHierarchyPreemptionDisabled(this);
+ } finally {
+ writeLock.unlock();
+ }
}
-
+
protected QueueInfo getQueueInfo() {
+ // Deliberately doesn't use lock here, because this method will be invoked
+ // from schedulerApplicationAttempt, to avoid deadlock, sacrifice
+ // consistency here.
+ // TODO, improve this
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(queueName);
queueInfo.setAccessibleNodeLabels(accessibleLabels);
@@ -318,8 +342,12 @@ public abstract class AbstractCSQueue implements CSQueue {
}
public QueueStatistics getQueueStatistics() {
- QueueStatistics stats =
- recordFactory.newRecordInstance(QueueStatistics.class);
+ // Deliberately doesn't use lock here, because this method will be invoked
+ // from schedulerApplicationAttempt, to avoid deadlock, sacrifice
+ // consistency here.
+ // TODO, improve this
+ QueueStatistics stats = recordFactory.newRecordInstance(
+ QueueStatistics.class);
stats.setNumAppsSubmitted(getMetrics().getAppsSubmitted());
stats.setNumAppsRunning(getMetrics().getAppsRunning());
stats.setNumAppsPending(getMetrics().getAppsPending());
@@ -351,26 +379,36 @@ public abstract class AbstractCSQueue implements CSQueue {
return minimumAllocation;
}
- synchronized void allocateResource(Resource clusterResource,
+ void allocateResource(Resource clusterResource,
Resource resource, String nodePartition, boolean changeContainerResource) {
- queueUsage.incUsed(nodePartition, resource);
+ try {
+ writeLock.lock();
+ queueUsage.incUsed(nodePartition, resource);
- if (!changeContainerResource) {
- ++numContainers;
+ if (!changeContainerResource) {
+ ++numContainers;
+ }
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ minimumAllocation, this, labelManager, nodePartition);
+ } finally {
+ writeLock.unlock();
}
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, nodePartition);
}
- protected synchronized void releaseResource(Resource clusterResource,
+ protected void releaseResource(Resource clusterResource,
Resource resource, String nodePartition, boolean changeContainerResource) {
- queueUsage.decUsed(nodePartition, resource);
+ try {
+ writeLock.lock();
+ queueUsage.decUsed(nodePartition, resource);
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, nodePartition);
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ minimumAllocation, this, labelManager, nodePartition);
- if (!changeContainerResource) {
- --numContainers;
+ if (!changeContainerResource) {
+ --numContainers;
+ }
+ } finally {
+ writeLock.unlock();
}
}
@@ -381,7 +419,13 @@ public abstract class AbstractCSQueue implements CSQueue {
@Private
public Map<AccessType, AccessControlList> getACLs() {
- return acls;
+ try {
+ readLock.lock();
+ return acls;
+ } finally {
+ readLock.unlock();
+ }
+
}
@Private
@@ -464,86 +508,88 @@ public abstract class AbstractCSQueue implements CSQueue {
minimumAllocation);
}
- synchronized boolean canAssignToThisQueue(Resource clusterResource,
+ boolean canAssignToThisQueue(Resource clusterResource,
String nodePartition, ResourceLimits currentResourceLimits,
Resource resourceCouldBeUnreserved, SchedulingMode schedulingMode) {
- // Get current limited resource:
- // - When doing RESPECT_PARTITION_EXCLUSIVITY allocation, we will respect
- // queues' max capacity.
- // - When doing IGNORE_PARTITION_EXCLUSIVITY allocation, we will not respect
- // queue's max capacity, queue's max capacity on the partition will be
- // considered to be 100%. Which is a queue can use all resource in the
- // partition.
- // Doing this because: for non-exclusive allocation, we make sure there's
- // idle resource on the partition, to avoid wastage, such resource will be
- // leveraged as much as we can, and preemption policy will reclaim it back
- // when partitoned-resource-request comes back.
- Resource currentLimitResource =
- getCurrentLimitResource(nodePartition, clusterResource,
- currentResourceLimits, schedulingMode);
-
- Resource nowTotalUsed = queueUsage.getUsed(nodePartition);
-
- // Set headroom for currentResourceLimits:
- // When queue is a parent queue: Headroom = limit - used + killable
- // When queue is a leaf queue: Headroom = limit - used (leaf queue cannot preempt itself)
- Resource usedExceptKillable = nowTotalUsed;
- if (null != getChildQueues() && !getChildQueues().isEmpty()) {
- usedExceptKillable = Resources.subtract(nowTotalUsed,
- getTotalKillableResource(nodePartition));
- }
- currentResourceLimits.setHeadroom(
- Resources.subtract(currentLimitResource, usedExceptKillable));
-
- if (Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
- usedExceptKillable, currentLimitResource)) {
-
- // if reservation continous looking enabled, check to see if could we
- // potentially use this node instead of a reserved node if the application
- // has reserved containers.
- // TODO, now only consider reservation cases when the node has no label
- if (this.reservationsContinueLooking
- && nodePartition.equals(RMNodeLabelsManager.NO_LABEL)
- && Resources.greaterThan(resourceCalculator, clusterResource,
- resourceCouldBeUnreserved, Resources.none())) {
- // resource-without-reserved = used - reserved
- Resource newTotalWithoutReservedResource =
- Resources.subtract(usedExceptKillable, resourceCouldBeUnreserved);
-
- // when total-used-without-reserved-resource < currentLimit, we still
- // have chance to allocate on this node by unreserving some containers
- if (Resources.lessThan(resourceCalculator, clusterResource,
- newTotalWithoutReservedResource, currentLimitResource)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("try to use reserved: " + getQueueName()
- + " usedResources: " + queueUsage.getUsed()
- + ", clusterResources: " + clusterResource
- + ", reservedResources: " + resourceCouldBeUnreserved
- + ", capacity-without-reserved: "
- + newTotalWithoutReservedResource + ", maxLimitCapacity: "
- + currentLimitResource);
+ try {
+ readLock.lock();
+ // Get current limited resource:
+ // - When doing RESPECT_PARTITION_EXCLUSIVITY allocation, we will respect
+ // queues' max capacity.
+ // - When doing IGNORE_PARTITION_EXCLUSIVITY allocation, we will not respect
+ // queue's max capacity, queue's max capacity on the partition will be
+ // considered to be 100%. Which is a queue can use all resource in the
+ // partition.
+ // Doing this because: for non-exclusive allocation, we make sure there's
+ // idle resource on the partition, to avoid wastage, such resource will be
+ // leveraged as much as we can, and preemption policy will reclaim it back
+ // when partitoned-resource-request comes back.
+ Resource currentLimitResource = getCurrentLimitResource(nodePartition,
+ clusterResource, currentResourceLimits, schedulingMode);
+
+ Resource nowTotalUsed = queueUsage.getUsed(nodePartition);
+
+ // Set headroom for currentResourceLimits:
+ // When queue is a parent queue: Headroom = limit - used + killable
+ // When queue is a leaf queue: Headroom = limit - used (leaf queue cannot preempt itself)
+ Resource usedExceptKillable = nowTotalUsed;
+ if (null != getChildQueues() && !getChildQueues().isEmpty()) {
+ usedExceptKillable = Resources.subtract(nowTotalUsed,
+ getTotalKillableResource(nodePartition));
+ }
+ currentResourceLimits.setHeadroom(
+ Resources.subtract(currentLimitResource, usedExceptKillable));
+
+ if (Resources.greaterThanOrEqual(resourceCalculator, clusterResource,
+ usedExceptKillable, currentLimitResource)) {
+
+ // if reservation continous looking enabled, check to see if could we
+ // potentially use this node instead of a reserved node if the application
+ // has reserved containers.
+ // TODO, now only consider reservation cases when the node has no label
+ if (this.reservationsContinueLooking && nodePartition.equals(
+ RMNodeLabelsManager.NO_LABEL) && Resources.greaterThan(
+ resourceCalculator, clusterResource, resourceCouldBeUnreserved,
+ Resources.none())) {
+ // resource-without-reserved = used - reserved
+ Resource newTotalWithoutReservedResource = Resources.subtract(
+ usedExceptKillable, resourceCouldBeUnreserved);
+
+ // when total-used-without-reserved-resource < currentLimit, we still
+ // have chance to allocate on this node by unreserving some containers
+ if (Resources.lessThan(resourceCalculator, clusterResource,
+ newTotalWithoutReservedResource, currentLimitResource)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "try to use reserved: " + getQueueName() + " usedResources: "
+ + queueUsage.getUsed() + ", clusterResources: "
+ + clusterResource + ", reservedResources: "
+ + resourceCouldBeUnreserved
+ + ", capacity-without-reserved: "
+ + newTotalWithoutReservedResource + ", maxLimitCapacity: "
+ + currentLimitResource);
+ }
+ return true;
}
- return true;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getQueueName() + "Check assign to queue, nodePartition="
+ + nodePartition + " usedResources: " + queueUsage
+ .getUsed(nodePartition) + " clusterResources: " + clusterResource
+ + " currentUsedCapacity " + Resources
+ .divide(resourceCalculator, clusterResource,
+ queueUsage.getUsed(nodePartition), labelManager
+ .getResourceByLabel(nodePartition, clusterResource))
+ + " max-capacity: " + queueCapacities
+ .getAbsoluteMaximumCapacity(nodePartition) + ")");
+ }
+ return false;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug(getQueueName()
- + "Check assign to queue, nodePartition="
- + nodePartition
- + " usedResources: "
- + queueUsage.getUsed(nodePartition)
- + " clusterResources: "
- + clusterResource
- + " currentUsedCapacity "
- + Resources.divide(resourceCalculator, clusterResource,
- queueUsage.getUsed(nodePartition),
- labelManager.getResourceByLabel(nodePartition, clusterResource))
- + " max-capacity: "
- + queueCapacities.getAbsoluteMaximumCapacity(nodePartition) + ")");
- }
- return false;
+ return true;
+ } finally {
+ readLock.unlock();
}
- return true;
+
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: HDFS-10879.
TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by
Xiao Chen.
Posted by ji...@apache.org.
HDFS-10879. TestEncryptionZonesWithKMS#testReadWrite fails intermittently. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e918dff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e918dff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e918dff
Branch: refs/heads/MAPREDUCE-6608
Commit: 0e918dff594e9ba5434fdee7fc1f6394b62b32cd
Parents: e80386d
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 20 16:52:05 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 20 16:56:52 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/TestEncryptionZones.java | 23 +++++++++++++++++++-
1 file changed, 22 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e918dff/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index b634dd2..9168ca6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -45,7 +45,9 @@ import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.crypto.key.kms.server.EagerKeyGeneratorKeyProviderCryptoExtension;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -734,14 +736,33 @@ public class TestEncryptionZones {
// Roll the key of the encryption zone
assertNumZones(1);
String keyName = dfsAdmin.listEncryptionZones().next().getKeyName();
+ FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1);
cluster.getNamesystem().getProvider().rollNewVersion(keyName);
+ /**
+ * due to the cache on the server side, client may get old keys.
+ * @see EagerKeyGeneratorKeyProviderCryptoExtension#rollNewVersion(String)
+ */
+ boolean rollSucceeded = false;
+ for (int i = 0; i <= EagerKeyGeneratorKeyProviderCryptoExtension
+ .KMS_KEY_CACHE_SIZE_DEFAULT + CommonConfigurationKeysPublic.
+ KMS_CLIENT_ENC_KEY_CACHE_SIZE_DEFAULT; ++i) {
+ KeyProviderCryptoExtension.EncryptedKeyVersion ekv2 =
+ cluster.getNamesystem().getProvider().generateEncryptedKey(TEST_KEY);
+ if (!(feInfo1.getEzKeyVersionName()
+ .equals(ekv2.getEncryptionKeyVersionName()))) {
+ rollSucceeded = true;
+ break;
+ }
+ }
+ Assert.assertTrue("rollover did not generate a new key even after"
+ + " queue is drained", rollSucceeded);
+
// Read them back in and compare byte-by-byte
verifyFilesEqual(fs, baseFile, encFile1, len);
// Write a new enc file and validate
final Path encFile2 = new Path(zone, "myfile2");
DFSTestUtil.createFile(fs, encFile2, len, (short) 1, 0xFEED);
// FEInfos should be different
- FileEncryptionInfo feInfo1 = getFileEncryptionInfo(encFile1);
FileEncryptionInfo feInfo2 = getFileEncryptionInfo(encFile2);
assertFalse("EDEKs should be different", Arrays
.equals(feInfo1.getEncryptedDataEncryptionKey(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-3692. Allow REST API to set a
user generated message when killing an application. Contributed by Rohith
Sharma K S
Posted by ji...@apache.org.
YARN-3692. Allow REST API to set a user generated message when killing an application. Contributed by Rohith Sharma K S
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0372dc6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0372dc6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0372dc6
Branch: refs/heads/MAPREDUCE-6608
Commit: d0372dc613136910160e9d42bd5eaa0d4bde2356
Parents: 5ffd4b7
Author: Naganarasimha <na...@apache.org>
Authored: Fri Sep 23 06:30:49 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Fri Sep 23 06:30:49 2016 +0530
----------------------------------------------------------------------
.../hadoop/mapred/ResourceMgrDelegate.java | 6 ++++++
.../protocolrecords/KillApplicationRequest.java | 18 ++++++++++++++++
.../src/main/proto/yarn_service_protos.proto | 1 +
.../hadoop/yarn/client/api/YarnClient.java | 14 +++++++++++++
.../yarn/client/api/impl/YarnClientImpl.java | 22 +++++++++++++++-----
.../impl/pb/KillApplicationRequestPBImpl.java | 18 ++++++++++++++++
.../server/resourcemanager/ClientRMService.java | 20 +++++++++++++-----
.../resourcemanager/webapp/RMWebServices.java | 8 +++++--
.../resourcemanager/webapp/dao/AppState.java | 8 +++++++
.../resourcemanager/TestClientRMService.java | 7 ++++++-
.../TestRMWebServicesAppsModification.java | 4 ++++
11 files changed, 113 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
index 159b518..c302553 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/ResourceMgrDelegate.java
@@ -511,4 +511,10 @@ public class ResourceMgrDelegate extends YarnClient {
throws YarnException, IOException {
client.signalToContainer(containerId, command);
}
+
+ @Override
+ public void killApplication(ApplicationId appId, String diagnostics)
+ throws YarnException, IOException {
+ client.killApplication(appId, diagnostics);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
index 606cf4e..a7679a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/KillApplicationRequest.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.protocolrecords;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Stable;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.util.Records;
@@ -57,4 +58,21 @@ public abstract class KillApplicationRequest {
@Public
@Stable
public abstract void setApplicationId(ApplicationId applicationId);
+
+ /**
+ * Get the <em>diagnostics</em> to which the application is being killed.
+ * @return <em>diagnostics</em> to which the application is being killed
+ */
+ @Public
+ @Unstable
+ public abstract String getDiagnostics();
+
+ /**
+ * Set the <em>diagnostics</em> to which the application is being killed.
+ * @param diagnostics <em>diagnostics</em> to which the application is being
+ * killed
+ */
+ @Public
+ @Unstable
+ public abstract void setDiagnostics(String diagnostics);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 97eaa5c..1385ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -156,6 +156,7 @@ message FailApplicationAttemptResponseProto {
message KillApplicationRequestProto {
optional ApplicationIdProto application_id = 1;
+ optional string diagnostics = 2;
}
message KillApplicationResponseProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 619ea0d..4cac2c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -174,6 +174,20 @@ public abstract class YarnClient extends AbstractService {
/**
* <p>
+ * Kill an application identified by given ID.
+ * </p>
+ * @param applicationId {@link ApplicationId} of the application that needs to
+ * be killed
+ * @param diagnostics for killing an application.
+ * @throws YarnException in case of errors or if YARN rejects the request due
+ * to access-control restrictions.
+ * @throws IOException
+ */
+ public abstract void killApplication(ApplicationId applicationId,
+ String diagnostics) throws YarnException, IOException;
+
+ /**
+ * <p>
* Get a report of the given Application.
* </p>
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index b91fde0..7760521 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -405,10 +405,21 @@ public class YarnClientImpl extends YarnClient {
@Override
public void killApplication(ApplicationId applicationId)
throws YarnException, IOException {
+ killApplication(applicationId, null);
+ }
+
+ @Override
+ public void killApplication(ApplicationId applicationId, String diagnostics)
+ throws YarnException, IOException {
+
KillApplicationRequest request =
Records.newRecord(KillApplicationRequest.class);
request.setApplicationId(applicationId);
+ if (diagnostics != null) {
+ request.setDiagnostics(diagnostics);
+ }
+
try {
int pollCount = 0;
long startTime = System.currentTimeMillis();
@@ -422,14 +433,15 @@ public class YarnClientImpl extends YarnClient {
}
long elapsedMillis = System.currentTimeMillis() - startTime;
- if (enforceAsyncAPITimeout() &&
- elapsedMillis >= this.asyncApiPollTimeoutMillis) {
- throw new YarnException("Timed out while waiting for application " +
- applicationId + " to be killed.");
+ if (enforceAsyncAPITimeout()
+ && elapsedMillis >= this.asyncApiPollTimeoutMillis) {
+ throw new YarnException("Timed out while waiting for application "
+ + applicationId + " to be killed.");
}
if (++pollCount % 10 == 0) {
- LOG.info("Waiting for application " + applicationId + " to be killed.");
+ LOG.info(
+ "Waiting for application " + applicationId + " to be killed.");
}
Thread.sleep(asyncApiPollIntervalMillis);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
index db97367..0520222 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/KillApplicationRequestPBImpl.java
@@ -127,6 +127,24 @@ public class KillApplicationRequestPBImpl extends KillApplicationRequest {
return ((ApplicationIdPBImpl)t).getProto();
}
+ @Override
+ public String getDiagnostics() {
+ KillApplicationRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasDiagnostics()) {
+ return null;
+ }
+ return (p.getDiagnostics());
+ }
+
+ @Override
+ public void setDiagnostics(String diagnostics) {
+ maybeInitBuilder();
+ if (diagnostics == null) {
+ builder.clearDiagnostics();
+ return;
+ }
+ builder.setDiagnostics(diagnostics);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 70e1863..e9bd230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -775,15 +775,25 @@ public class ClientRMService extends AbstractService implements
return KillApplicationResponse.newInstance(true);
}
- String message = "Kill application " + applicationId + " received from "
- + callerUGI;
+ StringBuilder message = new StringBuilder();
+ message.append("Application ").append(applicationId)
+ .append(" was killed by user ").append(callerUGI.getShortUserName());
+
InetAddress remoteAddress = Server.getRemoteIp();
if (null != remoteAddress) {
- message += " at " + remoteAddress.getHostAddress();
+ message.append(" at ").append(remoteAddress.getHostAddress());
+ }
+
+ String diagnostics = org.apache.commons.lang.StringUtils
+ .trimToNull(request.getDiagnostics());
+ if (diagnostics != null) {
+ message.append(" with diagnostic message: ");
+ message.append(diagnostics);
}
+
this.rmContext.getDispatcher().getEventHandler()
- .handle(new RMAppKillByClientEvent(applicationId, message, callerUGI,
- remoteAddress));
+ .handle(new RMAppKillByClientEvent(applicationId, message.toString(),
+ callerUGI, remoteAddress));
// For UnmanagedAMs, return true so they don't retry
return KillApplicationResponse.newInstance(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 4305fd5..ef6fd06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -925,7 +925,7 @@ public class RMWebServices extends WebServices {
// allow users to kill the app
if (targetState.getState().equals(YarnApplicationState.KILLED.toString())) {
- return killApp(app, callerUGI, hsr);
+ return killApp(app, callerUGI, hsr, targetState.getDiagnostics());
}
throw new BadRequestException("Only '"
+ YarnApplicationState.KILLED.toString()
@@ -1141,7 +1141,8 @@ public class RMWebServices extends WebServices {
}
protected Response killApp(RMApp app, UserGroupInformation callerUGI,
- HttpServletRequest hsr) throws IOException, InterruptedException {
+ HttpServletRequest hsr, String diagnostic)
+ throws IOException, InterruptedException {
if (app == null) {
throw new IllegalArgumentException("app cannot be null");
@@ -1158,6 +1159,9 @@ public class RMWebServices extends WebServices {
YarnException {
KillApplicationRequest req =
KillApplicationRequest.newInstance(appid);
+ if (diagnostic != null) {
+ req.setDiagnostics(diagnostic);
+ }
return rm.getClientRMService().forceKillApplication(req);
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
index e8f1cc3..533200e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppState.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement;
public class AppState {
String state;
+ private String diagnostics;
public AppState() {
}
@@ -43,4 +44,11 @@ public class AppState {
return this.state;
}
+ public String getDiagnostics() {
+ return diagnostics;
+ }
+
+ public void setDiagnostics(String diagnostics) {
+ this.diagnostics = diagnostics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 2a7971e..cee9086 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -506,7 +506,8 @@ public class TestClientRMService {
@Test
public void testForceKillApplication() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- MockRM rm = new MockRM();
+ conf.setBoolean(MockRM.ENABLE_WEBAPP, true);
+ MockRM rm = new MockRM(conf);
rm.init(conf);
rm.start();
@@ -522,6 +523,8 @@ public class TestClientRMService {
KillApplicationRequest killRequest1 =
KillApplicationRequest.newInstance(app1.getApplicationId());
+ String diagnostic = "message1";
+ killRequest1.setDiagnostics(diagnostic);
KillApplicationRequest killRequest2 =
KillApplicationRequest.newInstance(app2.getApplicationId());
@@ -539,6 +542,8 @@ public class TestClientRMService {
killAttemptCount > 1);
assertEquals("Incorrect number of apps in the RM", 1,
rmService.getApplications(getRequest).getApplicationList().size());
+ assertTrue("Diagnostic message is incorrect",
+ app1.getDiagnostics().toString().contains(diagnostic));
KillApplicationResponse killResponse2 =
rmService.forceKillApplication(killRequest2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0372dc6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index 53ef031..37e44d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -359,6 +359,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
{ MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
MediaType[] contentTypes =
{ MediaType.APPLICATION_JSON_TYPE, MediaType.APPLICATION_XML_TYPE };
+ String diagnostic = "message1";
for (String mediaType : mediaTypes) {
for (MediaType contentType : contentTypes) {
RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
@@ -366,6 +367,7 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
AppState targetState =
new AppState(YarnApplicationState.KILLED.toString());
+ targetState.setDiagnostics(diagnostic);
Object entity;
if (contentType.equals(MediaType.APPLICATION_JSON_TYPE)) {
@@ -425,6 +427,8 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
} else {
verifyAppStateXML(response, RMAppState.KILLED);
}
+ assertTrue("Diagnostic message is incorrect",
+ app.getDiagnostics().toString().contains(diagnostic));
break;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-5577. [Atsv2] Document object
passing in infofilters with an example (Rohith Sharma K S via Varun Saxena)
Posted by ji...@apache.org.
YARN-5577. [Atsv2] Document object passing in infofilters with an example (Rohith Sharma K S via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ea29e3bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ea29e3bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ea29e3bc
Branch: refs/heads/MAPREDUCE-6608
Commit: ea29e3bc27f15516f4346d1312eef703bcd3d032
Parents: 3552c2b
Author: Varun Saxena <va...@apache.org>
Authored: Mon Sep 19 14:33:06 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Mon Sep 19 14:33:06 2016 +0530
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ea29e3bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
index b6a0da4..6b7bd08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -712,6 +712,8 @@ none of the apps match the predicates, an empty list will be returned.
"eq" means equals, "ne" means not equals and existence of key is not required for a match and "ene" means not equals but existence of key is
required. We can combine any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.<br/>
_For example_ : infofilters can be "(((infokey1 eq value1) AND (infokey2 ne value1)) OR (infokey1 ene value3))".<br/>
+ Note : If value is an object then value can be given in the form of JSON format without any space.<br/>
+ _For example_ : infofilters can be (infokey1 eq {"<key>":"<value>","<key>":"<value>"...}).<br/>
Please note that URL unsafe characters such as spaces will have to be suitably encoded.
1. `conffilters` - If specified, matched applications must have exact matches to the given config name and must be either equal or not equal
to the given config value. Both the config name and value must be strings. conffilters are represented in the same form as infofilters.
@@ -837,6 +839,8 @@ match the predicates, an empty list will be returned.
"eq" means equals, "ne" means not equals and existence of key is not required for a match and "ene" means not equals but existence of key is
required. We can combine any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.<br/>
_For example_ : infofilters can be "(((infokey1 eq value1) AND (infokey2 ne value1)) OR (infokey1 ene value3))".<br/>
+ Note : If value is an object then value can be given in the form of JSON format without any space.<br/>
+ _For example_ : infofilters can be (infokey1 eq {"<key>":"<value>","<key>":"<value>"...}).<br/>
Please note that URL unsafe characters such as spaces will have to be suitably encoded.
1. `conffilters` - If specified, matched applications must have exact matches to the given config name and must be either equal or not equal
to the given config value. Both the config name and value must be strings. conffilters are represented in the same form as infofilters.
@@ -1035,6 +1039,8 @@ If none of the entities match the predicates, an empty list will be returned.
"eq" means equals, "ne" means not equals and existence of key is not required for a match and "ene" means not equals but existence of key is
required. We can combine any number of ANDs' and ORs' to create complex expressions. Brackets can be used to club expressions together.<br/>
_For example_ : infofilters can be "(((infokey1 eq value1) AND (infokey2 ne value1)) OR (infokey1 ene value3))".<br/>
+ Note : If value is an object then value can be given in the form of JSON format without any space.<br/>
+ _For example_ : infofilters can be (infokey1 eq {"<key>":"<value>","<key>":"<value>"...}).<br/>
Please note that URL unsafe characters such as spaces will have to be suitably encoded.
1. `conffilters` - If specified, matched entities must have exact matches to the given config name and must be either equal or not equal
to the given config value. Both the config name and value must be strings. conffilters are represented in the same form as infofilters.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: Revert "TimelineClient failed to
retry on java.net.SocketTimeoutException: Read timed out"
Posted by ji...@apache.org.
Revert "TimelineClient failed to retry on java.net.SocketTimeoutException: Read timed out"
This reverts commit 2e6ee957161ab63a02a7861b727efa6310b275b2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3293a7d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3293a7d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3293a7d9
Branch: refs/heads/MAPREDUCE-6608
Commit: 3293a7d92d563e45dd69be4ed60c01ec94af8a21
Parents: 2e6ee95
Author: Varun Saxena <va...@apache.org>
Authored: Fri Sep 23 13:25:46 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Sep 23 13:25:46 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3293a7d9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index dc4d3e6..56da2a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -265,8 +265,7 @@ public class TimelineClientImpl extends TimelineClient {
public boolean shouldRetryOn(Exception e) {
// Only retry on connection exceptions
return (e instanceof ClientHandlerException)
- && (e.getCause() instanceof ConnectException ||
- e.getCause() instanceof SocketTimeoutException);
+ && (e.getCause() instanceof ConnectException);
}
};
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: HDFS-10871. DiskBalancerWorkItem
should not import jackson relocated by htrace. Contributed by Manoj
Govindassamy.
Posted by ji...@apache.org.
HDFS-10871. DiskBalancerWorkItem should not import jackson relocated by htrace. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d85d9b2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d85d9b2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d85d9b2e
Branch: refs/heads/MAPREDUCE-6608
Commit: d85d9b2e7b4df04335111f4e3ce21a2fda39aee9
Parents: d0372dc
Author: Anu Engineer <ae...@apache.org>
Authored: Thu Sep 22 19:36:16 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Thu Sep 22 19:36:16 2016 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 4 ++++
.../hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java | 2 +-
.../hadoop/hdfs/server/diskbalancer/planner/MoveStep.java | 6 +-----
3 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d85d9b2e/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
index b8b531b..1e38019 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/pom.xml
@@ -107,6 +107,10 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<scope>test</scope>
<type>test-jar</type>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-annotations</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d85d9b2e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
index 592a89f..909bbd5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/datanode/DiskBalancerWorkItem.java
@@ -19,10 +19,10 @@
package org.apache.hadoop.hdfs.server.datanode;
+import com.fasterxml.jackson.annotation.JsonInclude;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.htrace.fasterxml.jackson.annotation.JsonInclude;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.map.ObjectReader;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d85d9b2e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/MoveStep.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/MoveStep.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/MoveStep.java
index b5f68fd..7f2f954 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/MoveStep.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/diskbalancer/planner/MoveStep.java
@@ -17,13 +17,9 @@
package org.apache.hadoop.hdfs.server.diskbalancer.planner;
+import com.fasterxml.jackson.annotation.JsonInclude;
import org.apache.hadoop.hdfs.server.diskbalancer.datamodel.DiskBalancerVolume;
import org.apache.hadoop.util.StringUtils;
-import org.apache.htrace.fasterxml.jackson.annotation.JsonInclude;
-
-
-
-
/**
* Ignore fields with default values. In most cases Throughtput, diskErrors
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-5642. Typos in 9 log
messages. Contributed by Mehran Hassani
Posted by ji...@apache.org.
YARN-5642. Typos in 9 log messages. Contributed by Mehran Hassani
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4174b975
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4174b975
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4174b975
Branch: refs/heads/MAPREDUCE-6608
Commit: 4174b9756c8c7877797545c4356b1f40df603ec5
Parents: 7d21c28
Author: Naganarasimha <na...@apache.org>
Authored: Sat Sep 17 10:35:39 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sat Sep 17 10:35:39 2016 +0530
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java | 2 +-
.../apache/hadoop/yarn/security/YarnAuthorizationProvider.java | 2 +-
.../FileSystemApplicationHistoryStore.java | 2 +-
.../containermanager/monitor/ContainersMonitorImpl.java | 2 +-
.../resourcemanager/reservation/AbstractReservationSystem.java | 2 +-
.../server/resourcemanager/scheduler/capacity/LeafQueue.java | 2 +-
.../server/resourcemanager/scheduler/capacity/ParentQueue.java | 4 ++--
.../server/resourcemanager/scheduler/fair/FairScheduler.java | 2 +-
8 files changed, 9 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 89b5861..df542ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -142,7 +142,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
protected void serviceStop() throws Exception {
if (drainEventsOnStop) {
blockNewEvents = true;
- LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
+ LOG.info("AsyncDispatcher is draining to stop, ignoring any new events.");
long endTime = System.currentTimeMillis() + getConfig()
.getLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT,
YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
index dd81ebd..4b43ea1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
@@ -54,7 +54,7 @@ public abstract class YarnAuthorizationProvider {
(YarnAuthorizationProvider) ReflectionUtils.newInstance(
authorizerClass, conf);
authorizer.init(conf);
- LOG.info(authorizerClass.getName() + " is instiantiated.");
+ LOG.info(authorizerClass.getName() + " is instantiated.");
}
}
return authorizer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
index 295b8ab..bb52b55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
@@ -405,7 +405,7 @@ public class FileSystemApplicationHistoryStore extends AbstractService
}
}
}
- LOG.info("Completed reading history information of all conatiners"
+ LOG.info("Completed reading history information of all containers"
+ " of application attempt " + appAttemptId);
} catch (IOException e) {
LOG.info("Error when reading history information of some containers"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index fb8cfc4..c456bde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -216,7 +216,7 @@ public class ContainersMonitorImpl extends AbstractService implements
return false;
}
if (!(isPmemCheckEnabled() || isVmemCheckEnabled())) {
- LOG.info("Neither virutal-memory nor physical-memory monitoring is " +
+ LOG.info("Neither virtual-memory nor physical-memory monitoring is " +
"needed. Not running the monitor-thread");
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
index 601a2a7..8769ca1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/AbstractReservationSystem.java
@@ -415,7 +415,7 @@ public abstract class AbstractReservationSystem extends AbstractService
minAllocation, maxAllocation, planQueueName,
getReplanner(planQueuePath), getReservationSchedulerConfiguration()
.getMoveOnExpiry(planQueuePath), rmContext);
- LOG.info("Intialized plan {} based on reservable queue {}",
+ LOG.info("Initialized plan {} based on reservable queue {}",
plan.toString(), planQueueName);
return plan;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 1ca69be..922d711 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -230,7 +230,7 @@ public class LeafQueue extends AbstractCSQueue {
LOG.info("Initializing " + queueName + "\n" +
"capacity = " + queueCapacities.getCapacity() +
" [= (float) configuredCapacity / 100 ]" + "\n" +
- "asboluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
+ "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
" [= parentAbsoluteCapacity * capacity ]" + "\n" +
"maxCapacity = " + queueCapacities.getMaximumCapacity() +
" [= configuredMaxCapacity ]" + "\n" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index a245e3b..3e9785f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -125,9 +125,9 @@ public class ParentQueue extends AbstractCSQueue {
LOG.info(queueName +
", capacity=" + this.queueCapacities.getCapacity() +
- ", asboluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
+ ", absoluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
", maxCapacity=" + this.queueCapacities.getMaximumCapacity() +
- ", asboluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() +
+ ", absoluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() +
", state=" + state +
", acls=" + aclsString +
", labels=" + labelStrBuilder.toString() + "\n" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4174b975/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 15e2318..310f2f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -944,7 +944,7 @@ public class FairScheduler extends
FSAppAttempt application = getSchedulerApp(appAttemptId);
if (application == null) {
LOG.info("Calling allocate on removed " +
- "or non existant application " + appAttemptId);
+ "or non existent application " + appAttemptId);
return EMPTY_ALLOCATION;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: HADOOP-13584. hdoop-aliyun: merge
HADOOP-12756 branch back.
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
new file mode 100644
index 0000000..88c83b5
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/site/markdown/tools/hadoop-aliyun/index.md
@@ -0,0 +1,294 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Hadoop-Aliyun module: Integration with Aliyun Web Services
+
+<!-- MACRO{toc|fromDepth=0|toDepth=5} -->
+
+## Overview
+
+The `hadoop-aliyun` module provides support for Aliyun integration with
+[Aliyun Object Storage Service (Aliyun OSS)](https://www.aliyun.com/product/oss).
+The generated JAR file, `hadoop-aliyun.jar` also declares a transitive
+dependency on all external artifacts which are needed for this support \u2014 enabling
+downstream applications to easily use this support.
+
+To make it part of Apache Hadoop's default classpath, simply make sure
+that HADOOP_OPTIONAL_TOOLS in hadoop-env.sh has 'hadoop-aliyun' in the list.
+
+### Features
+
+* Read and write data stored in Aliyun OSS.
+* Present a hierarchical file system view by implementing the standard Hadoop
+[`FileSystem`](../api/org/apache/hadoop/fs/FileSystem.html) interface.
+* Can act as a source of data in a MapReduce job, or a sink.
+
+### Warning #1: Object Stores are not filesystems.
+
+Aliyun OSS is an example of "an object store". In order to achieve scalability
+and especially high availability, Aliyun OSS has relaxed some of the constraints
+which classic "POSIX" filesystems promise.
+
+
+
+Specifically
+
+1. Atomic operations: `delete()` and `rename()` are implemented by recursive
+file-by-file operations. They take time at least proportional to the number of files,
+during which time partial updates may be visible. `delete()` and `rename()`
+can not guarantee atomicity. If the operations are interrupted, the filesystem
+is left in an intermediate state.
+2. File owner and group are persisted, but the permissions model is not enforced.
+Authorization occurs at the level of the entire Aliyun account via
+[Aliyun Resource Access Management (Aliyun RAM)](https://www.aliyun.com/product/ram).
+3. Directory last access time is not tracked.
+4. The append operation is not supported.
+
+### Warning #2: Directory last access time is not tracked,
+features of Hadoop relying on this can have unexpected behaviour. E.g. the
+AggregatedLogDeletionService of YARN will not remove the appropriate logfiles.
+
+### Warning #3: Your Aliyun credentials are valuable
+
+Your Aliyun credentials not only pay for services, they offer read and write
+access to the data. Anyone with the account can not only read your datasets
+\u2014they can delete them.
+
+Do not inadvertently share these credentials through means such as
+1. Checking in to SCM any configuration files containing the secrets.
+2. Logging them to a console, as they invariably end up being seen.
+3. Defining filesystem URIs with the credentials in the URL, such as
+`oss://accessKeyId:accessKeySecret@directory/file`. They will end up in
+logs and error messages.
+4. Including the secrets in bug reports.
+
+If you do any of these: change your credentials immediately!
+
+### Warning #4: The Aliyun OSS client provided by Aliyun E-MapReduce are different from this implementation
+
+Specifically: on Aliyun E-MapReduce, `oss://` is also supported but with
+a different implementation. If you are using Aliyun E-MapReduce,
+follow these instructions \u2014and be aware that all issues related to Aliyun
+OSS integration in E-MapReduce can only be addressed by Aliyun themselves:
+please raise your issues with them.
+
+## OSS
+
+### Authentication properties
+
+ <property>
+ <name>fs.oss.accessKeyId</name>
+ <description>Aliyun access key ID</description>
+ </property>
+
+ <property>
+ <name>fs.oss.accessKeySecret</name>
+ <description>Aliyun access key secret</description>
+ </property>
+
+ <property>
+ <name>fs.oss.credentials.provider</name>
+ <description>
+ Class name of a credentials provider that implements
+ com.aliyun.oss.common.auth.CredentialsProvider. Omit if using access/secret keys
+ or another authentication mechanism. The specified class must provide an
+ accessible constructor accepting java.net.URI and
+ org.apache.hadoop.conf.Configuration, or an accessible default constructor.
+ </description>
+ </property>
+
+### Other properties
+
+ <property>
+ <name>fs.oss.endpoint</name>
+ <description>Aliyun OSS endpoint to connect to. An up-to-date list is
+ provided in the Aliyun OSS Documentation.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.host</name>
+ <description>Hostname of the (optinal) proxy server for Aliyun OSS connection</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.port</name>
+ <description>Proxy server port</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.username</name>
+ <description>Username for authenticating with proxy server</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.password</name>
+ <description>Password for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.domain</name>
+ <description>Domain for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.proxy.workstation</name>
+ <description>Workstation for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.attempts.maximum</name>
+ <value>20</value>
+ <description>How many times we should retry commands on transient errors.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.establish.timeout</name>
+ <value>50000</value>
+ <description>Connection setup timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.timeout</name>
+ <value>200000</value>
+ <description>Socket connection timeout in milliseconds.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.paging.maximum</name>
+ <value>1000</value>
+ <description>How many keys to request from Aliyun OSS when doing directory listings at a time.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.upload.size</name>
+ <value>10485760</value>
+ <description>Size of each of multipart pieces in bytes.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.upload.threshold</name>
+ <value>20971520</value>
+ <description>Minimum size in bytes before we start a multipart uploads or copy.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400/value>
+ <description>Size in bytes in each request from ALiyun OSS.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.buffer.dir</name>
+ <description>Comma separated list of directories to buffer OSS data before uploading to Aliyun OSS</description>
+ </property>
+
+ <property>
+ <name>fs.oss.acl.default</name>
+ <value></vaule>
+ <description>Set a canned ACL for bucket. Value may be private, public-read, public-read-write.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.server-side-encryption-algorithm</name>
+ <value></vaule>
+ <description>Specify a server-side encryption algorithm for oss: file system.
+ Unset by default, and the only other currently allowable value is AES256.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.maximum</name>
+ <value>32</value>
+ <description>Number of simultaneous connections to oss.</description>
+ </property>
+
+ <property>
+ <name>fs.oss.connection.secure.enabled</name>
+ <value>true</value>
+ <description>Connect to oss over ssl or not, true by default.</description>
+ </property>
+
+## Testing the hadoop-aliyun Module
+
+To test `oss://` filesystem client, two files which pass in authentication
+details to the test runner are needed.
+
+1. `auth-keys.xml`
+2. `core-site.xml`
+
+Those two configuration files must be put into
+`hadoop-tools/hadoop-aliyun/src/test/resources`.
+
+### `core-site.xml`
+
+This file pre-exists and sources the configurations created in `auth-keys.xml`.
+
+For most cases, no modification is needed, unless a specific, non-default property
+needs to be set during the testing.
+
+### `auth-keys.xml`
+
+This file triggers the testing of Aliyun OSS module. Without this file,
+*none of the tests in this module will be executed*
+
+It contains the access key Id/secret and proxy information that are needed to
+connect to Aliyun OSS, and an OSS bucket URL should be also provided.
+
+1. `test.fs.oss.name` : the URL of the bucket for Aliyun OSS tests
+
+The contents of the bucket will be cleaned during the testing process, so
+do not use the bucket for any purpose other than testing.
+
+### Run Hadoop contract tests
+Create file `contract-test-options.xml` under `/test/resources`. If a
+specific file `fs.contract.test.fs.oss` test path is not defined, those
+tests will be skipped. Credentials are also needed to run any of those
+tests, they can be copied from `auth-keys.xml` or through direct
+XInclude inclusion. Here is an example of `contract-test-options.xml`:
+
+ <?xml version="1.0"?>
+ <?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+ <configuration>
+
+ <include xmlns="http://www.w3.org/2001/XInclude"
+ href="auth-keys.xml"/>
+
+ <property>
+ <name>fs.contract.test.fs.oss</name>
+ <value>oss://spark-tests</value>
+ </property>
+
+ <property>
+ <name>fs.oss.impl</name>
+ <value>org.apache.hadoop.fs.aliyun.AliyunOSSFileSystem</value>
+ </property>
+
+ <property>
+ <name>fs.oss.endpoint</name>
+ <value>oss-cn-hangzhou.aliyuncs.com</value>
+ </property>
+
+ <property>
+ <name>fs.oss.buffer.dir</name>
+ <value>/tmp/oss</value>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400</value>
+ </property>
+ </configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
new file mode 100644
index 0000000..901cb2b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSTestUtils.java
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.junit.internal.AssumptionViolatedException;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Utility class for Aliyun OSS Tests.
+ */
+public final class AliyunOSSTestUtils {
+
+ private AliyunOSSTestUtils() {
+ }
+
+ /**
+ * Create the test filesystem.
+ *
+ * If the test.fs.oss.name property is not set,
+ * tests will fail.
+ *
+ * @param conf configuration
+ * @return the FS
+ * @throws IOException
+ */
+ public static AliyunOSSFileSystem createTestFileSystem(Configuration conf)
+ throws IOException {
+ String fsname = conf.getTrimmed(
+ TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME, "");
+
+ boolean liveTest = StringUtils.isNotEmpty(fsname);
+ URI testURI = null;
+ if (liveTest) {
+ testURI = URI.create(fsname);
+ liveTest = testURI.getScheme().equals(Constants.FS_OSS);
+ }
+
+ if (!liveTest) {
+ throw new AssumptionViolatedException("No test filesystem in "
+ + TestAliyunOSSFileSystemContract.TEST_FS_OSS_NAME);
+ }
+ AliyunOSSFileSystem ossfs = new AliyunOSSFileSystem();
+ ossfs.initialize(testURI, conf);
+ return ossfs;
+ }
+
+ /**
+ * Generate unique test path for multiple user tests.
+ *
+ * @return root test path
+ */
+ public static String generateUniqueTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? "/test" :
+ "/" + testUniqueForkId + "/test";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java
new file mode 100644
index 0000000..e08a4dc
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunCredentials.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.common.auth.Credentials;
+import com.aliyun.oss.common.auth.InvalidCredentialsException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.aliyun.oss.contract.AliyunOSSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.apache.hadoop.fs.contract.AbstractFSContractTestBase;
+import org.junit.Test;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_ID;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.ACCESS_KEY_SECRET;
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SECURITY_TOKEN;
+
+/**
+ * Tests use of temporary credentials (for example, Aliyun STS & Aliyun OSS).
+ * This test extends a class that "does things to the root directory", and
+ * should only be used against transient filesystems where you don't care about
+ * the data.
+ */
+public class TestAliyunCredentials extends AbstractFSContractTestBase {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+ @Test
+ public void testCredentialMissingAccessKeyId() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY_ID, "");
+ conf.set(ACCESS_KEY_SECRET, "accessKeySecret");
+ conf.set(SECURITY_TOKEN, "token");
+ validateCredential(conf);
+ }
+
+ @Test
+ public void testCredentialMissingAccessKeySecret() throws Throwable {
+ Configuration conf = new Configuration();
+ conf.set(ACCESS_KEY_ID, "accessKeyId");
+ conf.set(ACCESS_KEY_SECRET, "");
+ conf.set(SECURITY_TOKEN, "token");
+ validateCredential(conf);
+ }
+
+ private void validateCredential(Configuration conf) {
+ try {
+ AliyunCredentialsProvider provider
+ = new AliyunCredentialsProvider(conf);
+ Credentials credentials = provider.getCredentials();
+ fail("Expected a CredentialInitializationException, got " + credentials);
+ } catch (InvalidCredentialsException expected) {
+ // expected
+ } catch (IOException e) {
+ fail("Unexpected exception.");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
new file mode 100644
index 0000000..ad8ef6e
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemContract.java
@@ -0,0 +1,239 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileAlreadyExistsException;
+import org.apache.hadoop.fs.FileSystemContractBaseTest;
+import org.apache.hadoop.fs.Path;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Tests a live Aliyun OSS system.
+ *
+ * This uses BlockJUnit4ClassRunner because FileSystemContractBaseTest from
+ * TestCase which uses the old Junit3 runner that doesn't ignore assumptions
+ * properly making it impossible to skip the tests if we don't have a valid
+ * bucket.
+ */
+public class TestAliyunOSSFileSystemContract
+ extends FileSystemContractBaseTest {
+ public static final String TEST_FS_OSS_NAME = "test.fs.oss.name";
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Override
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ super.setUp();
+ }
+
+ @Override
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(super.path(testRootPath), true);
+ }
+ super.tearDown();
+ }
+
+ @Override
+ protected Path path(String path) {
+ if (path.startsWith("/")) {
+ return super.path(testRootPath + path);
+ } else {
+ return super.path(testRootPath + "/" + path);
+ }
+ }
+
+ @Override
+ public void testMkdirsWithUmask() throws Exception {
+ // not supported
+ }
+
+ @Override
+ public void testRootDirAlwaysExists() throws Exception {
+ //this will throw an exception if the path is not found
+ fs.getFileStatus(super.path("/"));
+ //this catches overrides of the base exists() method that don't
+ //use getFileStatus() as an existence probe
+ assertTrue("FileSystem.exists() fails for root",
+ fs.exists(super.path("/")));
+ }
+
+ @Override
+ public void testRenameRootDirForbidden() throws Exception {
+ if (!renameSupported()) {
+ return;
+ }
+ rename(super.path("/"),
+ super.path("/test/newRootDir"),
+ false, true, false);
+ }
+
+ public void testDeleteSubdir() throws IOException {
+ Path parentDir = this.path("/test/hadoop");
+ Path file = this.path("/test/hadoop/file");
+ Path subdir = this.path("/test/hadoop/subdir");
+ this.createFile(file);
+
+ assertTrue("Created subdir", this.fs.mkdirs(subdir));
+ assertTrue("File exists", this.fs.exists(file));
+ assertTrue("Parent dir exists", this.fs.exists(parentDir));
+ assertTrue("Subdir exists", this.fs.exists(subdir));
+
+ assertTrue("Deleted subdir", this.fs.delete(subdir, true));
+ assertTrue("Parent should exist", this.fs.exists(parentDir));
+
+ assertTrue("Deleted file", this.fs.delete(file, false));
+ assertTrue("Parent should exist", this.fs.exists(parentDir));
+ }
+
+
+ @Override
+ protected boolean renameSupported() {
+ return true;
+ }
+
+ @Override
+ public void testRenameNonExistentPath() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/path");
+ Path dst = this.path("/test/new/newpath");
+ try {
+ super.rename(src, dst, false, false, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameFileMoveToNonExistentDirectory() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/file");
+ this.createFile(src);
+ Path dst = this.path("/test/new/newfile");
+ try {
+ super.rename(src, dst, false, true, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameDirectoryMoveToNonExistentDirectory() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/dir");
+ this.fs.mkdirs(src);
+ Path dst = this.path("/test/new/newdir");
+ try {
+ super.rename(src, dst, false, true, false);
+ fail("Should throw FileNotFoundException!");
+ } catch (FileNotFoundException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameFileMoveToExistingDirectory() throws Exception {
+ super.testRenameFileMoveToExistingDirectory();
+ }
+
+ @Override
+ public void testRenameFileAsExistingFile() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/file");
+ this.createFile(src);
+ Path dst = this.path("/test/new/newfile");
+ this.createFile(dst);
+ try {
+ super.rename(src, dst, false, true, true);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+ }
+
+ @Override
+ public void testRenameDirectoryAsExistingFile() throws Exception {
+ if (this.renameSupported()) {
+ Path src = this.path("/test/hadoop/dir");
+ this.fs.mkdirs(src);
+ Path dst = this.path("/test/new/newfile");
+ this.createFile(dst);
+ try {
+ super.rename(src, dst, false, true, true);
+ fail("Should throw FileAlreadyExistsException");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+ }
+
+ public void testGetFileStatusFileAndDirectory() throws Exception {
+ Path filePath = this.path("/test/oss/file1");
+ this.createFile(filePath);
+ assertTrue("Should be file", this.fs.getFileStatus(filePath).isFile());
+ assertFalse("Should not be directory",
+ this.fs.getFileStatus(filePath).isDirectory());
+
+ Path dirPath = this.path("/test/oss/dir");
+ this.fs.mkdirs(dirPath);
+ assertTrue("Should be directory",
+ this.fs.getFileStatus(dirPath).isDirectory());
+ assertFalse("Should not be file", this.fs.getFileStatus(dirPath).isFile());
+ }
+
+ public void testMkdirsForExistingFile() throws Exception {
+ Path testFile = this.path("/test/hadoop/file");
+ assertFalse(this.fs.exists(testFile));
+ this.createFile(testFile);
+ assertTrue(this.fs.exists(testFile));
+ try {
+ this.fs.mkdirs(testFile);
+ fail("Should throw FileAlreadyExistsException!");
+ } catch (FileAlreadyExistsException e) {
+ // expected
+ }
+ }
+
+ public void testWorkingDirectory() throws Exception {
+ Path workDir = super.path(this.getDefaultWorkingDirectory());
+ assertEquals(workDir, this.fs.getWorkingDirectory());
+ this.fs.setWorkingDirectory(super.path("."));
+ assertEquals(workDir, this.fs.getWorkingDirectory());
+ this.fs.setWorkingDirectory(super.path(".."));
+ assertEquals(workDir.getParent(), this.fs.getWorkingDirectory());
+ Path relativeDir = super.path("hadoop");
+ this.fs.setWorkingDirectory(relativeDir);
+ assertEquals(relativeDir, this.fs.getWorkingDirectory());
+ Path absoluteDir = super.path("/test/hadoop");
+ this.fs.setWorkingDirectory(absoluteDir);
+ assertEquals(absoluteDir, this.fs.getWorkingDirectory());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
new file mode 100644
index 0000000..7f4bac2
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSFileSystemStore.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.security.DigestInputStream;
+import java.security.DigestOutputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeNotNull;
+
+/**
+ * Test the bridging logic between Hadoop's abstract filesystem and
+ * Aliyun OSS.
+ */
+public class TestAliyunOSSFileSystemStore {
+ private Configuration conf;
+ private AliyunOSSFileSystemStore store;
+ private AliyunOSSFileSystem fs;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ fs = new AliyunOSSFileSystem();
+ fs.initialize(URI.create(conf.get("test.fs.oss.name")), conf);
+ store = fs.getStore();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ try {
+ store.purge("test");
+ } catch (Exception e) {
+ e.printStackTrace();
+ throw e;
+ }
+ }
+
+ @BeforeClass
+ public static void checkSettings() throws Exception {
+ Configuration conf = new Configuration();
+ assumeNotNull(conf.get(Constants.ACCESS_KEY_ID));
+ assumeNotNull(conf.get(Constants.ACCESS_KEY_SECRET));
+ assumeNotNull(conf.get("test.fs.oss.name"));
+ }
+
+ protected void writeRenameReadCompare(Path path, long len)
+ throws IOException, NoSuchAlgorithmException {
+ // If len > fs.oss.multipart.upload.threshold,
+ // we'll use a multipart upload copy
+ MessageDigest digest = MessageDigest.getInstance("MD5");
+ OutputStream out = new BufferedOutputStream(
+ new DigestOutputStream(fs.create(path, false), digest));
+ for (long i = 0; i < len; i++) {
+ out.write('Q');
+ }
+ out.flush();
+ out.close();
+
+ assertTrue("Exists", fs.exists(path));
+
+ Path copyPath = path.suffix(".copy");
+ fs.rename(path, copyPath);
+
+ assertTrue("Copy exists", fs.exists(copyPath));
+
+ // Download file from Aliyun OSS and compare the digest against the original
+ MessageDigest digest2 = MessageDigest.getInstance("MD5");
+ InputStream in = new BufferedInputStream(
+ new DigestInputStream(fs.open(copyPath), digest2));
+ long copyLen = 0;
+ while (in.read() != -1) {
+ copyLen++;
+ }
+ in.close();
+
+ assertEquals("Copy length matches original", len, copyLen);
+ assertArrayEquals("Digests match", digest.digest(), digest2.digest());
+ }
+
+ @Test
+ public void testSmallUpload() throws IOException, NoSuchAlgorithmException {
+ // Regular upload, regular copy
+ writeRenameReadCompare(new Path("/test/small"), 16384);
+ }
+
+ @Test
+ public void testLargeUpload()
+ throws IOException, NoSuchAlgorithmException {
+ // Multipart upload, multipart copy
+ writeRenameReadCompare(new Path("/test/xlarge"), 52428800L); // 50MB byte
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
new file mode 100644
index 0000000..37af28f
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
@@ -0,0 +1,145 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.io.IOUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Random;
+
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests basic functionality for AliyunOSSInputStream, including seeking and
+ * reading files.
+ */
+public class TestAliyunOSSInputStream {
+
+ private FileSystem fs;
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestAliyunOSSInputStream.class);
+
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ private Path setPath(String path) {
+ if (path.startsWith("/")) {
+ return new Path(testRootPath + path);
+ } else {
+ return new Path(testRootPath + "/" + path);
+ }
+ }
+
+ @Test
+ public void testSeekFile() throws Exception {
+ Path smallSeekFile = setPath("/test/smallSeekFile.txt");
+ long size = 5 * 1024 * 1024;
+
+ ContractTestUtils.generateTestFile(this.fs, smallSeekFile, size, 256, 255);
+ LOG.info("5MB file created: smallSeekFile.txt");
+
+ FSDataInputStream instream = this.fs.open(smallSeekFile);
+ int seekTimes = 5;
+ LOG.info("multiple fold position seeking test...:");
+ for (int i = 0; i < seekTimes; i++) {
+ long pos = size / (seekTimes - i) - 1;
+ LOG.info("begin seeking for pos: " + pos);
+ instream.seek(pos);
+ assertTrue("expected position at:" + pos + ", but got:"
+ + instream.getPos(), instream.getPos() == pos);
+ LOG.info("completed seeking at pos: " + instream.getPos());
+ }
+ LOG.info("random position seeking test...:");
+ Random rand = new Random();
+ for (int i = 0; i < seekTimes; i++) {
+ long pos = Math.abs(rand.nextLong()) % size;
+ LOG.info("begin seeking for pos: " + pos);
+ instream.seek(pos);
+ assertTrue("expected position at:" + pos + ", but got:"
+ + instream.getPos(), instream.getPos() == pos);
+ LOG.info("completed seeking at pos: " + instream.getPos());
+ }
+ IOUtils.closeStream(instream);
+ }
+
+ @Test
+ public void testReadFile() throws Exception {
+ final int bufLen = 256;
+ final int sizeFlag = 5;
+ String filename = "readTestFile_" + sizeFlag + ".txt";
+ Path readTestFile = setPath("/test/" + filename);
+ long size = sizeFlag * 1024 * 1024;
+
+ ContractTestUtils.generateTestFile(this.fs, readTestFile, size, 256, 255);
+ LOG.info(sizeFlag + "MB file created: /test/" + filename);
+
+ FSDataInputStream instream = this.fs.open(readTestFile);
+ byte[] buf = new byte[bufLen];
+ long bytesRead = 0;
+ while (bytesRead < size) {
+ int bytes;
+ if (size - bytesRead < bufLen) {
+ int remaining = (int)(size - bytesRead);
+ bytes = instream.read(buf, 0, remaining);
+ } else {
+ bytes = instream.read(buf, 0, bufLen);
+ }
+ bytesRead += bytes;
+
+ if (bytesRead % (1024 * 1024) == 0) {
+ int available = instream.available();
+ int remaining = (int)(size - bytesRead);
+ assertTrue("expected remaining:" + remaining + ", but got:" + available,
+ remaining == available);
+ LOG.info("Bytes read: " + Math.round((double)bytesRead / (1024 * 1024))
+ + " MB");
+ }
+ }
+ assertTrue(instream.available() == 0);
+ IOUtils.closeStream(instream);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
new file mode 100644
index 0000000..6b87d9c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
@@ -0,0 +1,91 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+
+/**
+ * Tests regular and multi-part upload functionality for AliyunOSSOutputStream.
+ */
+public class TestAliyunOSSOutputStream {
+ private FileSystem fs;
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
+ conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024);
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ protected Path getTestPath() {
+ return new Path(testRootPath + "/test-aliyun-oss");
+ }
+
+ @Test
+ public void testRegularUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
+ }
+
+ @Test
+ public void testMultiPartUploadLimit() throws IOException {
+ long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024);
+ assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024);
+ assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024,
+ 100 * 1024);
+ assert(10000 * 100 * 1024 / partSize3
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024,
+ 100 * 1024);
+ assert(10001 * 100 * 1024 / partSize4
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
new file mode 100644
index 0000000..624c606
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/AliyunOSSContract.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.AbstractBondedFSContract;
+
+/**
+ * The contract of Aliyun OSS: only enabled if the test bucket is provided.
+ */
+public class AliyunOSSContract extends AbstractBondedFSContract {
+
+ public static final String CONTRACT_XML = "contract/aliyun-oss.xml";
+
+ public AliyunOSSContract(Configuration conf) {
+ super(conf);
+ //insert the base features
+ addConfResource(CONTRACT_XML);
+ }
+
+ @Override
+ public String getScheme() {
+ return "oss";
+ }
+
+ @Override
+ public Path getTestPath() {
+ String testUniqueForkId = System.getProperty("test.unique.fork.id");
+ return testUniqueForkId == null ? super.getTestPath() :
+ new Path("/" + testUniqueForkId, "test");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java
new file mode 100644
index 0000000..88dd8cd
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractCreate.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractCreateTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract creating tests.
+ */
+public class TestAliyunOSSContractCreate extends AbstractContractCreateTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java
new file mode 100644
index 0000000..1658d80
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDelete.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractDeleteTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract deleting tests.
+ */
+public class TestAliyunOSSContractDelete extends AbstractContractDeleteTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
new file mode 100644
index 0000000..18d09d5
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
+
+/**
+ * Contract test suite covering Aliyun OSS integration with DistCp.
+ */
+public class TestAliyunOSSContractDistCp extends AbstractContractDistCpTest {
+
+ private static final long MULTIPART_SETTING = 8 * 1024 * 1024; // 8 MB
+
+ @Override
+ protected Configuration createConfiguration() {
+ Configuration newConf = super.createConfiguration();
+ newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING);
+ newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING);
+ return newConf;
+ }
+
+ @Override
+ protected AliyunOSSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java
new file mode 100644
index 0000000..c69124d
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractGetFileStatus.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Test getFileStatus and related listing operations.
+ */
+public class TestAliyunOSSContractGetFileStatus
+ extends AbstractContractGetFileStatusTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java
new file mode 100644
index 0000000..6cb7549
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractMkdir.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractMkdirTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract directory tests.
+ */
+public class TestAliyunOSSContractMkdir extends AbstractContractMkdirTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java
new file mode 100644
index 0000000..099aba6
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractOpen.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractOpenTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract opening file tests.
+ */
+public class TestAliyunOSSContractOpen extends AbstractContractOpenTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java
new file mode 100644
index 0000000..e15b3ba
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRename.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRenameTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract renaming tests.
+ */
+public class TestAliyunOSSContractRename extends AbstractContractRenameTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java
new file mode 100644
index 0000000..9faae37
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractRootDir.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+/**
+ * Root dir operations against an Aliyun OSS bucket.
+ */
+public class TestAliyunOSSContractRootDir extends
+ AbstractContractRootDirectoryTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestAliyunOSSContractRootDir.class);
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+
+ @Override
+ public void testListEmptyRootDirectory() throws IOException {
+ for (int attempt = 1, maxAttempts = 10; attempt <= maxAttempts; ++attempt) {
+ try {
+ super.testListEmptyRootDirectory();
+ break;
+ } catch (AssertionError | FileNotFoundException e) {
+ if (attempt < maxAttempts) {
+ LOG.info("Attempt {} of {} for empty root directory test failed. "
+ + "Attempting retry.", attempt, maxAttempts);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e2) {
+ Thread.currentThread().interrupt();
+ fail("Test interrupted.");
+ break;
+ }
+ } else {
+ LOG.error(
+ "Empty root directory test failed {} attempts. Failing test.",
+ maxAttempts);
+ throw e;
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java
new file mode 100644
index 0000000..b247ab1
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractSeek.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss.contract;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.contract.AbstractContractSeekTest;
+import org.apache.hadoop.fs.contract.AbstractFSContract;
+
+/**
+ * Aliyun OSS contract seeking tests.
+ */
+public class TestAliyunOSSContractSeek extends AbstractContractSeekTest {
+
+ @Override
+ protected AbstractFSContract createContract(Configuration conf) {
+ return new AliyunOSSContract(conf);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
new file mode 100644
index 0000000..7bbbf46
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/contract/aliyun-oss.xml
@@ -0,0 +1,115 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+ <property>
+ <name>fs.contract.test.random-seek-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-blobstore</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.is-case-sensitive</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-returns-false-if-source-missing</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-remove-dest-if-empty-dir</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-append</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-directory-delete</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-atomic-rename</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-block-locality</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-concat</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-seek-on-closed-file</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rejects-seek-past-eof</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-strict-exceptions</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-unix-permissions</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <name>fs.contract.rename-overwrites-dest</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.test.root-tests-enabled</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.contract.supports-getfilestatus</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>fs.oss.multipart.download.size</name>
+ <value>102400</value>
+ </property>
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml
new file mode 100644
index 0000000..fa4118c
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/core-site.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one
+ ~ or more contributor license agreements. See the NOTICE file
+ ~ distributed with this work for additional information
+ ~ regarding copyright ownership. The ASF licenses this file
+ ~ to you under the Apache License, Version 2.0 (the
+ ~ "License"); you may not use this file except in compliance
+ ~ with the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+<configuration>
+
+ <property>
+ <name>hadoop.tmp.dir</name>
+ <value>target/build/test</value>
+ <description>A base for other temporary directories.</description>
+ <final>true</final>
+ </property>
+
+ <!-- Turn security off for tests by default -->
+ <property>
+ <name>hadoop.security.authentication</name>
+ <value>simple</value>
+ </property>
+
+ <!--
+ To run these tests.
+
+ # Create a file auth-keys.xml - DO NOT ADD TO REVISION CONTROL
+ # add the property test.fs.oss.name to point to an OSS filesystem URL
+ # Add the credentials for the service you are testing against
+ -->
+ <include xmlns="http://www.w3.org/2001/XInclude" href="auth-keys.xml">
+ <fallback/>
+ </include>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
new file mode 100644
index 0000000..bb5cbe5
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
@@ -0,0 +1,23 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=INFO,stdout
+log4j.threshold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 899a945..14fa9f0 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -102,6 +102,12 @@
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-aliyun</artifactId>
+ <scope>compile</scope>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
<scope>compile</scope>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5707f88d/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index db002f4..e7e876b 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -47,6 +47,7 @@
<module>hadoop-aws</module>
<module>hadoop-kafka</module>
<module>hadoop-azure-datalake</module>
+ <module>hadoop-aliyun</module>
</modules>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDFS-10875. Optimize du -x to
cache intermediate result. Contributed by Xiao Chen.
Posted by ji...@apache.org.
HDFS-10875. Optimize du -x to cache intermediate result. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e52d6e7a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e52d6e7a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e52d6e7a
Branch: refs/heads/MAPREDUCE-6608
Commit: e52d6e7a46ceef74dd8d8a3d49c49420e3271365
Parents: 98bdb51
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Sep 19 21:44:42 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Sep 19 21:44:42 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/server/namenode/INodeDirectory.java | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e52d6e7a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index 9a8f9b2..24c8815 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -630,14 +630,15 @@ public class INodeDirectory extends INodeWithAdditionalFields
ContentSummaryComputationContext summary) {
final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
if (sf != null && snapshotId == Snapshot.CURRENT_STATE_ID) {
+ final ContentCounts counts = new ContentCounts.Builder().build();
// if the getContentSummary call is against a non-snapshot path, the
// computation should include all the deleted files/directories
sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
- summary.getCounts());
- // Also compute ContentSummary for snapshotCounts (So we can extract it
+ counts);
+ summary.getCounts().addContents(counts);
+ // Also add ContentSummary to snapshotCounts (So we can extract it
// later from the ContentSummary of all).
- sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
- summary.getSnapshotCounts());
+ summary.getSnapshotCounts().addContents(counts);
}
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null && snapshotId == Snapshot.CURRENT_STATE_ID) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-5663. Small refactor in
ZKRMStateStore. Contributed by Oleksii Dymytrov.
Posted by ji...@apache.org.
YARN-5663. Small refactor in ZKRMStateStore. Contributed by Oleksii Dymytrov.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14a696f3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14a696f3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14a696f3
Branch: refs/heads/MAPREDUCE-6608
Commit: 14a696f369f7e3802587f57c8fff3aa51b5ab576
Parents: 5707f88
Author: Akira Ajisaka <aa...@apache.org>
Authored: Mon Sep 26 15:00:01 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Mon Sep 26 15:00:01 2016 +0900
----------------------------------------------------------------------
.../server/resourcemanager/recovery/ZKRMStateStore.java | 9 +++------
1 file changed, 3 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a696f3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 9e05f6d..c24b3e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -748,17 +748,14 @@ public class ZKRMStateStore extends RMStateStore {
String nodeCreatePath =
getNodePath(dtMasterKeysRootPath, DELEGATION_KEY_PREFIX
+ delegationKey.getKeyId());
- ByteArrayOutputStream os = new ByteArrayOutputStream();
- DataOutputStream fsOut = new DataOutputStream(os);
if (LOG.isDebugEnabled()) {
LOG.debug("Storing RMDelegationKey_" + delegationKey.getKeyId());
}
- delegationKey.write(fsOut);
- try {
+ ByteArrayOutputStream os = new ByteArrayOutputStream();
+ try(DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
safeCreate(nodeCreatePath, os.toByteArray(), zkAcl,
CreateMode.PERSISTENT);
- } finally {
- os.close();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: HADOOP-13643. Math error in
AbstractContractDistCpTest. Contributed by Aaron Fabbri.
Posted by ji...@apache.org.
HADOOP-13643. Math error in AbstractContractDistCpTest. Contributed by Aaron Fabbri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5ef51e7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5ef51e7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5ef51e7
Branch: refs/heads/MAPREDUCE-6608
Commit: e5ef51e717647328db9f2b80f21fe44b99079d08
Parents: b8a2d7b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Sep 23 10:00:32 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Sep 23 10:01:30 2016 +0100
----------------------------------------------------------------------
.../apache/hadoop/tools/contract/AbstractContractDistCpTest.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5ef51e7/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
index a4f50c7..21a14d3 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/contract/AbstractContractDistCpTest.java
@@ -160,7 +160,7 @@ public abstract class AbstractContractDistCpTest
Path inputFile3 = new Path(inputDir, "file3");
mkdirs(srcFS, inputDir);
int fileSizeKb = conf.getInt("scale.test.distcp.file.size.kb", 10 * 1024);
- int fileSizeMb = fileSizeKb * 1024;
+ int fileSizeMb = fileSizeKb / 1024;
getLog().info("{} with file size {}", testName.getMethodName(), fileSizeMb);
byte[] data1 = dataset((fileSizeMb + 1) * 1024 * 1024, 33, 43);
createFile(srcFS, inputFile1, true, data1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-5540. Scheduler spends too
much time looking at empty priorities. Contributed by Jason Lowe
Posted by ji...@apache.org.
YARN-5540. Scheduler spends too much time looking at empty priorities. Contributed by Jason Lowe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7558dbbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7558dbbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7558dbbb
Branch: refs/heads/MAPREDUCE-6608
Commit: 7558dbbb481eab055e794beb3603bbe5671a4b4c
Parents: c54f6ef
Author: Jason Lowe <jl...@apache.org>
Authored: Mon Sep 19 20:31:35 2016 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Mon Sep 19 20:31:35 2016 +0000
----------------------------------------------------------------------
.../scheduler/AppSchedulingInfo.java | 96 +++++++++++---------
.../scheduler/TestAppSchedulingInfo.java | 65 +++++++++++++
2 files changed, 118 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7558dbbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index c677345..39820f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -26,8 +26,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
-import java.util.TreeSet;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
@@ -59,7 +59,6 @@ import org.apache.hadoop.yarn.util.resource.Resources;
public class AppSchedulingInfo {
private static final Log LOG = LogFactory.getLog(AppSchedulingInfo.class);
- private static final int EPOCH_BIT_SHIFT = 40;
private final ApplicationId applicationId;
private final ApplicationAttemptId applicationAttemptId;
@@ -79,7 +78,8 @@ public class AppSchedulingInfo {
private Set<String> requestedPartitions = new HashSet<>();
- final Set<SchedulerRequestKey> schedulerKeys = new TreeSet<>();
+ private final ConcurrentSkipListMap<SchedulerRequestKey, Integer>
+ schedulerKeys = new ConcurrentSkipListMap<>();
final Map<SchedulerRequestKey, Map<String, ResourceRequest>>
resourceRequestMap = new ConcurrentHashMap<>();
final Map<NodeId, Map<SchedulerRequestKey, Map<ContainerId,
@@ -236,6 +236,7 @@ public class AppSchedulingInfo {
if (null == requestsOnNodeWithPriority) {
requestsOnNodeWithPriority = new TreeMap<>();
requestsOnNode.put(schedulerKey, requestsOnNodeWithPriority);
+ incrementSchedulerKeyReference(schedulerKey);
}
requestsOnNodeWithPriority.put(containerId, request);
@@ -250,11 +251,30 @@ public class AppSchedulingInfo {
LOG.debug("Added increase request:" + request.getContainerId()
+ " delta=" + delta);
}
-
- // update Scheduler Keys
- schedulerKeys.add(schedulerKey);
}
-
+
+ private void incrementSchedulerKeyReference(
+ SchedulerRequestKey schedulerKey) {
+ Integer schedulerKeyCount = schedulerKeys.get(schedulerKey);
+ if (schedulerKeyCount == null) {
+ schedulerKeys.put(schedulerKey, 1);
+ } else {
+ schedulerKeys.put(schedulerKey, schedulerKeyCount + 1);
+ }
+ }
+
+ private void decrementSchedulerKeyReference(
+ SchedulerRequestKey schedulerKey) {
+ Integer schedulerKeyCount = schedulerKeys.get(schedulerKey);
+ if (schedulerKeyCount != null) {
+ if (schedulerKeyCount > 1) {
+ schedulerKeys.put(schedulerKey, schedulerKeyCount - 1);
+ } else {
+ schedulerKeys.remove(schedulerKey);
+ }
+ }
+ }
+
public synchronized boolean removeIncreaseRequest(NodeId nodeId,
SchedulerRequestKey schedulerKey, ContainerId containerId) {
Map<SchedulerRequestKey, Map<ContainerId, SchedContainerChangeRequest>>
@@ -275,6 +295,7 @@ public class AppSchedulingInfo {
// remove hierarchies if it becomes empty
if (requestsOnNodeWithPriority.isEmpty()) {
requestsOnNode.remove(schedulerKey);
+ decrementSchedulerKeyReference(schedulerKey);
}
if (requestsOnNode.isEmpty()) {
containerIncreaseRequestMap.remove(nodeId);
@@ -341,7 +362,6 @@ public class AppSchedulingInfo {
if (asks == null) {
asks = new ConcurrentHashMap<>();
this.resourceRequestMap.put(schedulerKey, asks);
- this.schedulerKeys.add(schedulerKey);
}
// Increment number of containers if recovering preempted resources
@@ -360,29 +380,34 @@ public class AppSchedulingInfo {
anyResourcesUpdated = true;
- // Activate application. Metrics activation is done here.
- // TODO: Shouldn't we activate even if numContainers = 0?
- if (request.getNumContainers() > 0) {
- activeUsersManager.activateApplication(user, applicationId);
- }
-
// Update pendingResources
- updatePendingResources(lastRequest, request, queue.getMetrics());
+ updatePendingResources(lastRequest, request, schedulerKey,
+ queue.getMetrics());
}
}
return anyResourcesUpdated;
}
private void updatePendingResources(ResourceRequest lastRequest,
- ResourceRequest request, QueueMetrics metrics) {
+ ResourceRequest request, SchedulerRequestKey schedulerKey,
+ QueueMetrics metrics) {
+ int lastRequestContainers =
+ (lastRequest != null) ? lastRequest.getNumContainers() : 0;
if (request.getNumContainers() <= 0) {
+ if (lastRequestContainers >= 0) {
+ decrementSchedulerKeyReference(schedulerKey);
+ }
LOG.info("checking for deactivate of application :"
+ this.applicationId);
checkForDeactivation();
+ } else {
+ // Activate application. Metrics activation is done here.
+ if (lastRequestContainers <= 0) {
+ incrementSchedulerKeyReference(schedulerKey);
+ activeUsersManager.activateApplication(user, applicationId);
+ }
}
- int lastRequestContainers =
- (lastRequest != null) ? lastRequest.getNumContainers() : 0;
Resource lastRequestCapability =
lastRequest != null ? lastRequest.getCapability() : Resources.none();
metrics.incrPendingResources(user,
@@ -505,7 +530,7 @@ public class AppSchedulingInfo {
}
public synchronized Collection<SchedulerRequestKey> getSchedulerKeys() {
- return schedulerKeys;
+ return schedulerKeys.keySet();
}
public synchronized Map<String, ResourceRequest> getResourceRequests(
@@ -617,7 +642,7 @@ public class AppSchedulingInfo {
} else if (type == NodeType.RACK_LOCAL) {
allocateRackLocal(node, schedulerKey, request, resourceRequests);
} else {
- allocateOffSwitch(request, resourceRequests);
+ allocateOffSwitch(request, resourceRequests, schedulerKey);
}
QueueMetrics metrics = queue.getMetrics();
if (pending) {
@@ -656,7 +681,7 @@ public class AppSchedulingInfo {
ResourceRequest offRackRequest = resourceRequestMap.get(schedulerKey).get(
ResourceRequest.ANY);
- decrementOutstanding(offRackRequest);
+ decrementOutstanding(offRackRequest, schedulerKey);
// Update cloned NodeLocal, RackLocal and OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(nodeLocalRequest));
@@ -684,7 +709,7 @@ public class AppSchedulingInfo {
ResourceRequest offRackRequest = resourceRequestMap.get(schedulerKey).get(
ResourceRequest.ANY);
- decrementOutstanding(offRackRequest);
+ decrementOutstanding(offRackRequest, schedulerKey);
// Update cloned RackLocal and OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(rackLocalRequest));
@@ -696,15 +721,16 @@ public class AppSchedulingInfo {
* application.
*/
private synchronized void allocateOffSwitch(
- ResourceRequest offSwitchRequest, List<ResourceRequest> resourceRequests) {
+ ResourceRequest offSwitchRequest, List<ResourceRequest> resourceRequests,
+ SchedulerRequestKey schedulerKey) {
// Update future requirements
- decrementOutstanding(offSwitchRequest);
+ decrementOutstanding(offSwitchRequest, schedulerKey);
// Update cloned OffRack requests for recovery
resourceRequests.add(cloneResourceRequest(offSwitchRequest));
}
private synchronized void decrementOutstanding(
- ResourceRequest offSwitchRequest) {
+ ResourceRequest offSwitchRequest, SchedulerRequestKey schedulerKey) {
int numOffSwitchContainers = offSwitchRequest.getNumContainers() - 1;
// Do not remove ANY
@@ -713,6 +739,7 @@ public class AppSchedulingInfo {
// Do we have any outstanding requests?
// If there is nothing, we need to deactivate this application
if (numOffSwitchContainers == 0) {
+ decrementSchedulerKeyReference(schedulerKey);
checkForDeactivation();
}
@@ -723,24 +750,7 @@ public class AppSchedulingInfo {
}
private synchronized void checkForDeactivation() {
- boolean deactivate = true;
- for (SchedulerRequestKey schedulerKey : getSchedulerKeys()) {
- ResourceRequest request =
- getResourceRequest(schedulerKey, ResourceRequest.ANY);
- if (request != null) {
- if (request.getNumContainers() > 0) {
- deactivate = false;
- break;
- }
- }
- }
-
- // also we need to check increase request
- if (!deactivate) {
- deactivate = containerIncreaseRequestMap.isEmpty();
- }
-
- if (deactivate) {
+ if (schedulerKeys.isEmpty()) {
activeUsersManager.deactivateApplication(user, applicationId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7558dbbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
index 503ea34..7f9c719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAppSchedulingInfo.java
@@ -23,11 +23,14 @@ import static org.mockito.Mockito.mock;
import java.util.ArrayList;
import java.util.Iterator;
+import java.util.List;
import java.util.TreeSet;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.TestUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FSLeafQueue;
import org.junit.Assert;
@@ -105,4 +108,66 @@ public class TestAppSchedulingInfo {
Assert.assertEquals(2, sk.getPriority().getPriority());
Assert.assertEquals(6, sk.getAllocationRequestId());
}
+
+ @Test
+ public void testSchedulerKeyAccounting() {
+ ApplicationId appIdImpl = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appIdImpl, 1);
+
+ Queue queue = mock(Queue.class);
+ doReturn(mock(QueueMetrics.class)).when(queue).getMetrics();
+ AppSchedulingInfo info = new AppSchedulingInfo(
+ appAttemptId, "test", queue, mock(ActiveUsersManager.class), 0,
+ new ResourceUsage());
+ Assert.assertEquals(0, info.getSchedulerKeys().size());
+
+ Priority pri1 = Priority.newInstance(1);
+ ResourceRequest req1 = ResourceRequest.newInstance(pri1,
+ ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
+ Priority pri2 = Priority.newInstance(2);
+ ResourceRequest req2 = ResourceRequest.newInstance(pri2,
+ ResourceRequest.ANY, Resource.newInstance(1024, 1), 2);
+ List<ResourceRequest> reqs = new ArrayList<>();
+ reqs.add(req1);
+ reqs.add(req2);
+ info.updateResourceRequests(reqs, false);
+ ArrayList<SchedulerRequestKey> keys =
+ new ArrayList<>(info.getSchedulerKeys());
+ Assert.assertEquals(2, keys.size());
+ Assert.assertEquals(SchedulerRequestKey.create(req1), keys.get(0));
+ Assert.assertEquals(SchedulerRequestKey.create(req2), keys.get(1));
+
+ // iterate to verify no ConcurrentModificationException
+ for (SchedulerRequestKey schedulerKey : info.getSchedulerKeys()) {
+ info.allocate(NodeType.OFF_SWITCH, null, schedulerKey, req1, null);
+ }
+ Assert.assertEquals(1, info.getSchedulerKeys().size());
+ Assert.assertEquals(SchedulerRequestKey.create(req2),
+ info.getSchedulerKeys().iterator().next());
+
+ req2 = ResourceRequest.newInstance(pri2,
+ ResourceRequest.ANY, Resource.newInstance(1024, 1), 1);
+ reqs.clear();
+ reqs.add(req2);
+ info.updateResourceRequests(reqs, false);
+ info.allocate(NodeType.OFF_SWITCH, null, SchedulerRequestKey.create(req2),
+ req2, null);
+ Assert.assertEquals(0, info.getSchedulerKeys().size());
+
+ req1 = ResourceRequest.newInstance(pri1,
+ ResourceRequest.ANY, Resource.newInstance(1024, 1), 5);
+ reqs.clear();
+ reqs.add(req1);
+ info.updateResourceRequests(reqs, false);
+ Assert.assertEquals(1, info.getSchedulerKeys().size());
+ Assert.assertEquals(SchedulerRequestKey.create(req1),
+ info.getSchedulerKeys().iterator().next());
+ req1 = ResourceRequest.newInstance(pri1,
+ ResourceRequest.ANY, Resource.newInstance(1024, 1), 0);
+ reqs.clear();
+ reqs.add(req1);
+ info.updateResourceRequests(reqs, false);
+ Assert.assertEquals(0, info.getSchedulerKeys().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-3140. Improve locks in
AbstractCSQueue/LeafQueue/ParentQueue. Contributed by Wangda Tan
Posted by ji...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b66d9ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 922d711..6129772 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
@@ -85,11 +86,11 @@ public class LeafQueue extends AbstractCSQueue {
private static final Log LOG = LogFactory.getLog(LeafQueue.class);
private float absoluteUsedCapacity = 0.0f;
- private int userLimit;
- private float userLimitFactor;
+ private volatile int userLimit;
+ private volatile float userLimitFactor;
protected int maxApplications;
- protected int maxApplicationsPerUser;
+ protected volatile int maxApplicationsPerUser;
private float maxAMResourcePerQueuePercent;
@@ -97,15 +98,15 @@ public class LeafQueue extends AbstractCSQueue {
private volatile boolean rackLocalityFullReset;
Map<ApplicationAttemptId, FiCaSchedulerApp> applicationAttemptMap =
- new HashMap<ApplicationAttemptId, FiCaSchedulerApp>();
+ new ConcurrentHashMap<>();
private Priority defaultAppPriorityPerQueue;
- private OrderingPolicy<FiCaSchedulerApp> pendingOrderingPolicy = null;
+ private final OrderingPolicy<FiCaSchedulerApp> pendingOrderingPolicy;
private volatile float minimumAllocationFactor;
- private Map<String, User> users = new HashMap<String, User>();
+ private Map<String, User> users = new ConcurrentHashMap<>();
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -122,7 +123,7 @@ public class LeafQueue extends AbstractCSQueue {
private volatile ResourceLimits cachedResourceLimitsForHeadroom = null;
- private OrderingPolicy<FiCaSchedulerApp> orderingPolicy = null;
+ private volatile OrderingPolicy<FiCaSchedulerApp> orderingPolicy = null;
// Summation of consumed ratios for all users in queue
private float totalUserConsumedRatio = 0;
@@ -131,7 +132,7 @@ public class LeafQueue extends AbstractCSQueue {
// record all ignore partition exclusivityRMContainer, this will be used to do
// preemption, key is the partition of the RMContainer allocated on
private Map<String, TreeSet<RMContainer>> ignorePartitionExclusivityRMContainers =
- new HashMap<>();
+ new ConcurrentHashMap<>();
@SuppressWarnings({ "unchecked", "rawtypes" })
public LeafQueue(CapacitySchedulerContext cs,
@@ -154,125 +155,125 @@ public class LeafQueue extends AbstractCSQueue {
setupQueueConfigs(cs.getClusterResource());
}
- protected synchronized void setupQueueConfigs(Resource clusterResource)
+ protected void setupQueueConfigs(Resource clusterResource)
throws IOException {
- super.setupQueueConfigs(clusterResource);
-
- this.lastClusterResource = clusterResource;
-
- this.cachedResourceLimitsForHeadroom = new ResourceLimits(clusterResource);
-
- // Initialize headroom info, also used for calculating application
- // master resource limits. Since this happens during queue initialization
- // and all queues may not be realized yet, we'll use (optimistic)
- // absoluteMaxCapacity (it will be replaced with the more accurate
- // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
- setQueueResourceLimitsInfo(clusterResource);
+ try {
+ writeLock.lock();
+ super.setupQueueConfigs(clusterResource);
- CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ this.lastClusterResource = clusterResource;
- setOrderingPolicy(conf.<FiCaSchedulerApp>getOrderingPolicy(getQueuePath()));
+ this.cachedResourceLimitsForHeadroom = new ResourceLimits(
+ clusterResource);
- userLimit = conf.getUserLimit(getQueuePath());
- userLimitFactor = conf.getUserLimitFactor(getQueuePath());
+ // Initialize headroom info, also used for calculating application
+ // master resource limits. Since this happens during queue initialization
+ // and all queues may not be realized yet, we'll use (optimistic)
+ // absoluteMaxCapacity (it will be replaced with the more accurate
+ // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
+ setQueueResourceLimitsInfo(clusterResource);
- maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath());
- if (maxApplications < 0) {
- int maxSystemApps = conf.getMaximumSystemApplications();
- maxApplications =
- (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity());
- }
- maxApplicationsPerUser = Math.min(maxApplications,
- (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor));
-
- maxAMResourcePerQueuePercent =
- conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
-
- if (!SchedulerUtils.checkQueueLabelExpression(
- this.accessibleLabels, this.defaultLabelExpression, null)) {
- throw new IOException("Invalid default label expression of "
- + " queue="
- + getQueueName()
- + " doesn't have permission to access all labels "
- + "in default label expression. labelExpression of resource request="
- + (this.defaultLabelExpression == null ? ""
- : this.defaultLabelExpression)
- + ". Queue labels="
- + (getAccessibleNodeLabels() == null ? "" : StringUtils.join(
- getAccessibleNodeLabels().iterator(), ',')));
- }
-
- nodeLocalityDelay = conf.getNodeLocalityDelay();
- rackLocalityFullReset = conf.getRackLocalityFullReset();
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
- // re-init this since max allocation could have changed
- this.minimumAllocationFactor =
- Resources.ratio(resourceCalculator,
- Resources.subtract(maximumAllocation, minimumAllocation),
- maximumAllocation);
+ setOrderingPolicy(
+ conf.<FiCaSchedulerApp>getOrderingPolicy(getQueuePath()));
- StringBuilder aclsString = new StringBuilder();
- for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
- aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
- }
+ userLimit = conf.getUserLimit(getQueuePath());
+ userLimitFactor = conf.getUserLimitFactor(getQueuePath());
- StringBuilder labelStrBuilder = new StringBuilder();
- if (accessibleLabels != null) {
- for (String s : accessibleLabels) {
- labelStrBuilder.append(s);
- labelStrBuilder.append(",");
+ maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath());
+ if (maxApplications < 0) {
+ int maxSystemApps = conf.getMaximumSystemApplications();
+ maxApplications =
+ (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity());
}
- }
+ maxApplicationsPerUser = Math.min(maxApplications,
+ (int) (maxApplications * (userLimit / 100.0f) * userLimitFactor));
+
+ maxAMResourcePerQueuePercent =
+ conf.getMaximumApplicationMasterResourcePerQueuePercent(
+ getQueuePath());
+
+ if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
+ this.defaultLabelExpression, null)) {
+ throw new IOException(
+ "Invalid default label expression of " + " queue=" + getQueueName()
+ + " doesn't have permission to access all labels "
+ + "in default label expression. labelExpression of resource request="
+ + (this.defaultLabelExpression == null ?
+ "" :
+ this.defaultLabelExpression) + ". Queue labels=" + (
+ getAccessibleNodeLabels() == null ?
+ "" :
+ StringUtils
+ .join(getAccessibleNodeLabels().iterator(), ',')));
+ }
+
+ nodeLocalityDelay = conf.getNodeLocalityDelay();
+ rackLocalityFullReset = conf.getRackLocalityFullReset();
+
+ // re-init this since max allocation could have changed
+ this.minimumAllocationFactor = Resources.ratio(resourceCalculator,
+ Resources.subtract(maximumAllocation, minimumAllocation),
+ maximumAllocation);
- defaultAppPriorityPerQueue = Priority.newInstance(conf
- .getDefaultApplicationPriorityConfPerQueue(getQueuePath()));
-
- LOG.info("Initializing " + queueName + "\n" +
- "capacity = " + queueCapacities.getCapacity() +
- " [= (float) configuredCapacity / 100 ]" + "\n" +
- "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
- " [= parentAbsoluteCapacity * capacity ]" + "\n" +
- "maxCapacity = " + queueCapacities.getMaximumCapacity() +
- " [= configuredMaxCapacity ]" + "\n" +
- "absoluteMaxCapacity = " + queueCapacities.getAbsoluteMaximumCapacity() +
- " [= 1.0 maximumCapacity undefined, " +
- "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" +
- "\n" +
- "userLimit = " + userLimit +
- " [= configuredUserLimit ]" + "\n" +
- "userLimitFactor = " + userLimitFactor +
- " [= configuredUserLimitFactor ]" + "\n" +
- "maxApplications = " + maxApplications +
- " [= configuredMaximumSystemApplicationsPerQueue or" +
- " (int)(configuredMaximumSystemApplications * absoluteCapacity)]" +
- "\n" +
- "maxApplicationsPerUser = " + maxApplicationsPerUser +
- " [= (int)(maxApplications * (userLimit / 100.0f) * " +
- "userLimitFactor) ]" + "\n" +
- "usedCapacity = " + queueCapacities.getUsedCapacity() +
- " [= usedResourcesMemory / " +
- "(clusterResourceMemory * absoluteCapacity)]" + "\n" +
- "absoluteUsedCapacity = " + absoluteUsedCapacity +
- " [= usedResourcesMemory / clusterResourceMemory]" + "\n" +
- "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent +
- " [= configuredMaximumAMResourcePercent ]" + "\n" +
- "minimumAllocationFactor = " + minimumAllocationFactor +
- " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / " +
- "maximumAllocationMemory ]" + "\n" +
- "maximumAllocation = " + maximumAllocation +
- " [= configuredMaxAllocation ]" + "\n" +
- "numContainers = " + numContainers +
- " [= currentNumContainers ]" + "\n" +
- "state = " + state +
- " [= configuredState ]" + "\n" +
- "acls = " + aclsString +
- " [= configuredAcls ]" + "\n" +
- "nodeLocalityDelay = " + nodeLocalityDelay + "\n" +
- "labels=" + labelStrBuilder.toString() + "\n" +
- "reservationsContinueLooking = " +
- reservationsContinueLooking + "\n" +
- "preemptionDisabled = " + getPreemptionDisabled() + "\n" +
- "defaultAppPriorityPerQueue = " + defaultAppPriorityPerQueue);
+ StringBuilder aclsString = new StringBuilder();
+ for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
+ aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
+ }
+
+ StringBuilder labelStrBuilder = new StringBuilder();
+ if (accessibleLabels != null) {
+ for (String s : accessibleLabels) {
+ labelStrBuilder.append(s);
+ labelStrBuilder.append(",");
+ }
+ }
+
+ defaultAppPriorityPerQueue = Priority.newInstance(
+ conf.getDefaultApplicationPriorityConfPerQueue(getQueuePath()));
+
+ LOG.info(
+ "Initializing " + queueName + "\n" + "capacity = " + queueCapacities
+ .getCapacity() + " [= (float) configuredCapacity / 100 ]" + "\n"
+ + "absoluteCapacity = " + queueCapacities.getAbsoluteCapacity()
+ + " [= parentAbsoluteCapacity * capacity ]" + "\n"
+ + "maxCapacity = " + queueCapacities.getMaximumCapacity()
+ + " [= configuredMaxCapacity ]" + "\n" + "absoluteMaxCapacity = "
+ + queueCapacities.getAbsoluteMaximumCapacity()
+ + " [= 1.0 maximumCapacity undefined, "
+ + "(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]"
+ + "\n" + "userLimit = " + userLimit + " [= configuredUserLimit ]"
+ + "\n" + "userLimitFactor = " + userLimitFactor
+ + " [= configuredUserLimitFactor ]" + "\n" + "maxApplications = "
+ + maxApplications
+ + " [= configuredMaximumSystemApplicationsPerQueue or"
+ + " (int)(configuredMaximumSystemApplications * absoluteCapacity)]"
+ + "\n" + "maxApplicationsPerUser = " + maxApplicationsPerUser
+ + " [= (int)(maxApplications * (userLimit / 100.0f) * "
+ + "userLimitFactor) ]" + "\n" + "usedCapacity = "
+ + queueCapacities.getUsedCapacity() + " [= usedResourcesMemory / "
+ + "(clusterResourceMemory * absoluteCapacity)]" + "\n"
+ + "absoluteUsedCapacity = " + absoluteUsedCapacity
+ + " [= usedResourcesMemory / clusterResourceMemory]" + "\n"
+ + "maxAMResourcePerQueuePercent = " + maxAMResourcePerQueuePercent
+ + " [= configuredMaximumAMResourcePercent ]" + "\n"
+ + "minimumAllocationFactor = " + minimumAllocationFactor
+ + " [= (float)(maximumAllocationMemory - minimumAllocationMemory) / "
+ + "maximumAllocationMemory ]" + "\n" + "maximumAllocation = "
+ + maximumAllocation + " [= configuredMaxAllocation ]" + "\n"
+ + "numContainers = " + numContainers
+ + " [= currentNumContainers ]" + "\n" + "state = " + state
+ + " [= configuredState ]" + "\n" + "acls = " + aclsString
+ + " [= configuredAcls ]" + "\n" + "nodeLocalityDelay = "
+ + nodeLocalityDelay + "\n" + "labels=" + labelStrBuilder
+ .toString() + "\n" + "reservationsContinueLooking = "
+ + reservationsContinueLooking + "\n" + "preemptionDisabled = "
+ + getPreemptionDisabled() + "\n" + "defaultAppPriorityPerQueue = "
+ + defaultAppPriorityPerQueue);
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
@@ -300,7 +301,7 @@ public class LeafQueue extends AbstractCSQueue {
return maxApplications;
}
- public synchronized int getMaxApplicationsPerUser() {
+ public int getMaxApplicationsPerUser() {
return maxApplicationsPerUser;
}
@@ -318,7 +319,8 @@ public class LeafQueue extends AbstractCSQueue {
* Set user limit - used only for testing.
* @param userLimit new user limit
*/
- synchronized void setUserLimit(int userLimit) {
+ @VisibleForTesting
+ void setUserLimit(int userLimit) {
this.userLimit = userLimit;
}
@@ -326,50 +328,74 @@ public class LeafQueue extends AbstractCSQueue {
* Set user limit factor - used only for testing.
* @param userLimitFactor new user limit factor
*/
- synchronized void setUserLimitFactor(float userLimitFactor) {
+ @VisibleForTesting
+ void setUserLimitFactor(float userLimitFactor) {
this.userLimitFactor = userLimitFactor;
}
@Override
- public synchronized int getNumApplications() {
- return getNumPendingApplications() + getNumActiveApplications();
- }
-
- public synchronized int getNumPendingApplications() {
- return pendingOrderingPolicy.getNumSchedulableEntities();
+ public int getNumApplications() {
+ try {
+ readLock.lock();
+ return getNumPendingApplications() + getNumActiveApplications();
+ } finally {
+ readLock.unlock();
+ }
}
- public synchronized int getNumActiveApplications() {
- return orderingPolicy.getNumSchedulableEntities();
+ public int getNumPendingApplications() {
+ try {
+ readLock.lock();
+ return pendingOrderingPolicy.getNumSchedulableEntities();
+ } finally {
+ readLock.unlock();
+ }
}
- @Private
- public synchronized int getNumApplications(String user) {
- return getUser(user).getTotalApplications();
+ public int getNumActiveApplications() {
+ try {
+ readLock.lock();
+ return orderingPolicy.getNumSchedulableEntities();
+ } finally {
+ readLock.unlock();
+ }
}
@Private
- public synchronized int getNumPendingApplications(String user) {
- return getUser(user).getPendingApplications();
+ public int getNumPendingApplications(String user) {
+ try {
+ readLock.lock();
+ User u = getUser(user);
+ if (null == u) {
+ return 0;
+ }
+ return u.getPendingApplications();
+ } finally {
+ readLock.unlock();
+ }
}
@Private
- public synchronized int getNumActiveApplications(String user) {
- return getUser(user).getActiveApplications();
- }
-
- @Override
- public synchronized QueueState getState() {
- return state;
+ public int getNumActiveApplications(String user) {
+ try {
+ readLock.lock();
+ User u = getUser(user);
+ if (null == u) {
+ return 0;
+ }
+ return u.getActiveApplications();
+ } finally {
+ readLock.unlock();
+ }
}
@Private
- public synchronized int getUserLimit() {
+ public int getUserLimit() {
return userLimit;
}
@Private
- public synchronized float getUserLimitFactor() {
+ public float getUserLimitFactor() {
return userLimitFactor;
}
@@ -381,112 +407,145 @@ public class LeafQueue extends AbstractCSQueue {
}
@Override
- public synchronized List<QueueUserACLInfo>
+ public List<QueueUserACLInfo>
getQueueUserAclInfo(UserGroupInformation user) {
- QueueUserACLInfo userAclInfo =
- recordFactory.newRecordInstance(QueueUserACLInfo.class);
- List<QueueACL> operations = new ArrayList<QueueACL>();
- for (QueueACL operation : QueueACL.values()) {
- if (hasAccess(operation, user)) {
- operations.add(operation);
+ try {
+ readLock.lock();
+ QueueUserACLInfo userAclInfo = recordFactory.newRecordInstance(
+ QueueUserACLInfo.class);
+ List<QueueACL> operations = new ArrayList<>();
+ for (QueueACL operation : QueueACL.values()) {
+ if (hasAccess(operation, user)) {
+ operations.add(operation);
+ }
}
+
+ userAclInfo.setQueueName(getQueueName());
+ userAclInfo.setUserAcls(operations);
+ return Collections.singletonList(userAclInfo);
+ } finally {
+ readLock.unlock();
}
- userAclInfo.setQueueName(getQueueName());
- userAclInfo.setUserAcls(operations);
- return Collections.singletonList(userAclInfo);
}
public String toString() {
- return queueName + ": " +
- "capacity=" + queueCapacities.getCapacity() + ", " +
- "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
- "usedResources=" + queueUsage.getUsed() + ", " +
- "usedCapacity=" + getUsedCapacity() + ", " +
- "absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
- "numApps=" + getNumApplications() + ", " +
- "numContainers=" + getNumContainers();
+ try {
+ readLock.lock();
+ return queueName + ": " + "capacity=" + queueCapacities.getCapacity()
+ + ", " + "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity()
+ + ", " + "usedResources=" + queueUsage.getUsed() + ", "
+ + "usedCapacity=" + getUsedCapacity() + ", " + "absoluteUsedCapacity="
+ + getAbsoluteUsedCapacity() + ", " + "numApps=" + getNumApplications()
+ + ", " + "numContainers=" + getNumContainers();
+ } finally {
+ readLock.unlock();
+ }
+
}
-
+
@VisibleForTesting
- public synchronized void setNodeLabelManager(RMNodeLabelsManager mgr) {
- this.labelManager = mgr;
+ public User getUser(String userName) {
+ return users.get(userName);
}
- @VisibleForTesting
- public synchronized User getUser(String userName) {
- User user = users.get(userName);
- if (user == null) {
- user = new User();
- users.put(userName, user);
+ // Get and add user if absent
+ private User getUserAndAddIfAbsent(String userName) {
+ try {
+ writeLock.lock();
+ User u = users.get(userName);
+ if (null == u) {
+ u = new User();
+ users.put(userName, u);
+ }
+ return u;
+ } finally {
+ writeLock.unlock();
}
- return user;
}
/**
* @return an ArrayList of UserInfo objects who are active in this queue
*/
- public synchronized ArrayList<UserInfo> getUsers() {
- ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
- for (Map.Entry<String, User> entry : users.entrySet()) {
- User user = entry.getValue();
- usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(user
- .getAllUsed()), user.getActiveApplications(), user
- .getPendingApplications(), Resources.clone(user
- .getConsumedAMResources()), Resources.clone(user
- .getUserResourceLimit()), user.getResourceUsage()));
+ public ArrayList<UserInfo> getUsers() {
+ try {
+ readLock.lock();
+ ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
+ for (Map.Entry<String, User> entry : users.entrySet()) {
+ User user = entry.getValue();
+ usersToReturn.add(
+ new UserInfo(entry.getKey(), Resources.clone(user.getAllUsed()),
+ user.getActiveApplications(), user.getPendingApplications(),
+ Resources.clone(user.getConsumedAMResources()),
+ Resources.clone(user.getUserResourceLimit()),
+ user.getResourceUsage()));
+ }
+ return usersToReturn;
+ } finally {
+ readLock.unlock();
}
- return usersToReturn;
}
@Override
- public synchronized void reinitialize(
+ public void reinitialize(
CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException {
- // Sanity check
- if (!(newlyParsedQueue instanceof LeafQueue) ||
- !newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
- throw new IOException("Trying to reinitialize " + getQueuePath() +
- " from " + newlyParsedQueue.getQueuePath());
- }
-
- LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;
+ try {
+ writeLock.lock();
+ // Sanity check
+ if (!(newlyParsedQueue instanceof LeafQueue) || !newlyParsedQueue
+ .getQueuePath().equals(getQueuePath())) {
+ throw new IOException(
+ "Trying to reinitialize " + getQueuePath() + " from "
+ + newlyParsedQueue.getQueuePath());
+ }
- // don't allow the maximum allocation to be decreased in size
- // since we have already told running AM's the size
- Resource oldMax = getMaximumAllocation();
- Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
- if (newMax.getMemorySize() < oldMax.getMemorySize()
- || newMax.getVirtualCores() < oldMax.getVirtualCores()) {
- throw new IOException(
- "Trying to reinitialize "
- + getQueuePath()
- + " the maximum allocation size can not be decreased!"
- + " Current setting: " + oldMax
- + ", trying to set it to: " + newMax);
- }
+ LeafQueue newlyParsedLeafQueue = (LeafQueue) newlyParsedQueue;
+
+ // don't allow the maximum allocation to be decreased in size
+ // since we have already told running AM's the size
+ Resource oldMax = getMaximumAllocation();
+ Resource newMax = newlyParsedLeafQueue.getMaximumAllocation();
+ if (newMax.getMemorySize() < oldMax.getMemorySize()
+ || newMax.getVirtualCores() < oldMax.getVirtualCores()) {
+ throw new IOException("Trying to reinitialize " + getQueuePath()
+ + " the maximum allocation size can not be decreased!"
+ + " Current setting: " + oldMax + ", trying to set it to: "
+ + newMax);
+ }
- setupQueueConfigs(clusterResource);
+ setupQueueConfigs(clusterResource);
- // queue metrics are updated, more resource may be available
- // activate the pending applications if possible
- activateApplications();
+ // queue metrics are updated, more resource may be available
+ // activate the pending applications if possible
+ activateApplications();
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
public void submitApplicationAttempt(FiCaSchedulerApp application,
String userName) {
// Careful! Locking order is important!
- synchronized (this) {
- User user = getUser(userName);
+ try {
+ writeLock.lock();
+
+ // TODO, should use getUser, use this method just to avoid UT failure
+ // which is caused by wrong invoking order, will fix UT separately
+ User user = getUserAndAddIfAbsent(userName);
+
// Add the attempt to our data-structures
addApplicationAttempt(application, user);
+ } finally {
+ writeLock.unlock();
}
// We don't want to update metrics for move app
if (application.isPending()) {
metrics.submitAppAttempt(userName);
}
+
getParent().submitApplicationAttempt(application, userName);
}
@@ -494,37 +553,38 @@ public class LeafQueue extends AbstractCSQueue {
public void submitApplication(ApplicationId applicationId, String userName,
String queue) throws AccessControlException {
// Careful! Locking order is important!
-
- User user = null;
- synchronized (this) {
-
+ try {
+ writeLock.lock();
// Check if the queue is accepting jobs
if (getState() != QueueState.RUNNING) {
- String msg = "Queue " + getQueuePath() +
- " is STOPPED. Cannot accept submission of application: " + applicationId;
+ String msg = "Queue " + getQueuePath()
+ + " is STOPPED. Cannot accept submission of application: "
+ + applicationId;
LOG.info(msg);
throw new AccessControlException(msg);
}
// Check submission limits for queues
if (getNumApplications() >= getMaxApplications()) {
- String msg = "Queue " + getQueuePath() +
- " already has " + getNumApplications() + " applications," +
- " cannot accept submission of application: " + applicationId;
+ String msg =
+ "Queue " + getQueuePath() + " already has " + getNumApplications()
+ + " applications,"
+ + " cannot accept submission of application: " + applicationId;
LOG.info(msg);
throw new AccessControlException(msg);
}
// Check submission limits for the user on this queue
- user = getUser(userName);
+ User user = getUserAndAddIfAbsent(userName);
if (user.getTotalApplications() >= getMaxApplicationsPerUser()) {
- String msg = "Queue " + getQueuePath() +
- " already has " + user.getTotalApplications() +
- " applications from user " + userName +
- " cannot accept submission of application: " + applicationId;
+ String msg = "Queue " + getQueuePath() + " already has " + user
+ .getTotalApplications() + " applications from user " + userName
+ + " cannot accept submission of application: " + applicationId;
LOG.info(msg);
throw new AccessControlException(msg);
}
+ } finally {
+ writeLock.unlock();
}
// Inform the parent queue
@@ -546,214 +606,237 @@ public class LeafQueue extends AbstractCSQueue {
return queueUsage.getAMLimit(nodePartition);
}
- public synchronized Resource calculateAndGetAMResourceLimit() {
+ @VisibleForTesting
+ public Resource calculateAndGetAMResourceLimit() {
return calculateAndGetAMResourceLimitPerPartition(
RMNodeLabelsManager.NO_LABEL);
}
@VisibleForTesting
- public synchronized Resource getUserAMResourceLimit() {
+ public Resource getUserAMResourceLimit() {
return getUserAMResourceLimitPerPartition(RMNodeLabelsManager.NO_LABEL);
}
- public synchronized Resource getUserAMResourceLimitPerPartition(
+ public Resource getUserAMResourceLimitPerPartition(
String nodePartition) {
- /*
- * The user am resource limit is based on the same approach as the user
- * limit (as it should represent a subset of that). This means that it uses
- * the absolute queue capacity (per partition) instead of the max and is
- * modified by the userlimit and the userlimit factor as is the userlimit
- */
- float effectiveUserLimit = Math.max(userLimit / 100.0f,
- 1.0f / Math.max(getActiveUsersManager().getNumActiveUsers(), 1));
-
- Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition), minimumAllocation);
-
- Resource userAMLimit = Resources.multiplyAndNormalizeUp(resourceCalculator,
- queuePartitionResource,
- queueCapacities.getMaxAMResourcePercentage(nodePartition)
- * effectiveUserLimit * userLimitFactor, minimumAllocation);
- return Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
- userAMLimit, getAMResourceLimitPerPartition(nodePartition))
- ? userAMLimit
- : getAMResourceLimitPerPartition(nodePartition);
- }
-
- public synchronized Resource calculateAndGetAMResourceLimitPerPartition(
+ try {
+ readLock.lock();
+ /*
+ * The user am resource limit is based on the same approach as the user
+ * limit (as it should represent a subset of that). This means that it uses
+ * the absolute queue capacity (per partition) instead of the max and is
+ * modified by the userlimit and the userlimit factor as is the userlimit
+ */
+ float effectiveUserLimit = Math.max(userLimit / 100.0f,
+ 1.0f / Math.max(getActiveUsersManager().getNumActiveUsers(), 1));
+
+ Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
+ resourceCalculator,
+ labelManager.getResourceByLabel(nodePartition, lastClusterResource),
+ queueCapacities.getAbsoluteCapacity(nodePartition),
+ minimumAllocation);
+
+ Resource userAMLimit = Resources.multiplyAndNormalizeUp(
+ resourceCalculator, queuePartitionResource,
+ queueCapacities.getMaxAMResourcePercentage(nodePartition)
+ * effectiveUserLimit * userLimitFactor, minimumAllocation);
+ return Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
+ userAMLimit, getAMResourceLimitPerPartition(nodePartition)) ?
+ userAMLimit :
+ getAMResourceLimitPerPartition(nodePartition);
+ } finally {
+ readLock.unlock();
+ }
+
+ }
+
+ public Resource calculateAndGetAMResourceLimitPerPartition(
String nodePartition) {
- /*
- * For non-labeled partition, get the max value from resources currently
- * available to the queue and the absolute resources guaranteed for the
- * partition in the queue. For labeled partition, consider only the absolute
- * resources guaranteed. Multiply this value (based on labeled/
- * non-labeled), * with per-partition am-resource-percent to get the max am
- * resource limit for this queue and partition.
- */
- Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- labelManager.getResourceByLabel(nodePartition, lastClusterResource),
- queueCapacities.getAbsoluteCapacity(nodePartition), minimumAllocation);
-
- Resource queueCurrentLimit = Resources.none();
- // For non-labeled partition, we need to consider the current queue
- // usage limit.
- if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
- synchronized (queueResourceLimitsInfo) {
- queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit();
+ try {
+ writeLock.lock();
+ /*
+ * For non-labeled partition, get the max value from resources currently
+ * available to the queue and the absolute resources guaranteed for the
+ * partition in the queue. For labeled partition, consider only the absolute
+ * resources guaranteed. Multiply this value (based on labeled/
+ * non-labeled), * with per-partition am-resource-percent to get the max am
+ * resource limit for this queue and partition.
+ */
+ Resource queuePartitionResource = Resources.multiplyAndNormalizeUp(
+ resourceCalculator,
+ labelManager.getResourceByLabel(nodePartition, lastClusterResource),
+ queueCapacities.getAbsoluteCapacity(nodePartition),
+ minimumAllocation);
+
+ Resource queueCurrentLimit = Resources.none();
+ // For non-labeled partition, we need to consider the current queue
+ // usage limit.
+ if (nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
+ synchronized (queueResourceLimitsInfo){
+ queueCurrentLimit = queueResourceLimitsInfo.getQueueCurrentLimit();
+ }
}
- }
- float amResourcePercent = queueCapacities
- .getMaxAMResourcePercentage(nodePartition);
+ float amResourcePercent = queueCapacities.getMaxAMResourcePercentage(
+ nodePartition);
- // Current usable resource for this queue and partition is the max of
- // queueCurrentLimit and queuePartitionResource.
- Resource queuePartitionUsableResource = Resources.max(resourceCalculator,
- lastClusterResource, queueCurrentLimit, queuePartitionResource);
+ // Current usable resource for this queue and partition is the max of
+ // queueCurrentLimit and queuePartitionResource.
+ Resource queuePartitionUsableResource = Resources.max(resourceCalculator,
+ lastClusterResource, queueCurrentLimit, queuePartitionResource);
- Resource amResouceLimit = Resources.multiplyAndNormalizeUp(
- resourceCalculator, queuePartitionUsableResource, amResourcePercent,
- minimumAllocation);
+ Resource amResouceLimit = Resources.multiplyAndNormalizeUp(
+ resourceCalculator, queuePartitionUsableResource, amResourcePercent,
+ minimumAllocation);
- metrics.setAMResouceLimit(amResouceLimit);
- queueUsage.setAMLimit(nodePartition, amResouceLimit);
- return amResouceLimit;
+ metrics.setAMResouceLimit(amResouceLimit);
+ queueUsage.setAMLimit(nodePartition, amResouceLimit);
+ return amResouceLimit;
+ } finally {
+ writeLock.unlock();
+ }
}
- private synchronized void activateApplications() {
- // limit of allowed resource usage for application masters
- Map<String, Resource> userAmPartitionLimit =
- new HashMap<String, Resource>();
-
- // AM Resource Limit for accessible labels can be pre-calculated.
- // This will help in updating AMResourceLimit for all labels when queue
- // is initialized for the first time (when no applications are present).
- for (String nodePartition : getNodeLabelsForQueue()) {
- calculateAndGetAMResourceLimitPerPartition(nodePartition);
- }
+ private void activateApplications() {
+ try {
+ writeLock.lock();
+ // limit of allowed resource usage for application masters
+ Map<String, Resource> userAmPartitionLimit =
+ new HashMap<String, Resource>();
+
+ // AM Resource Limit for accessible labels can be pre-calculated.
+ // This will help in updating AMResourceLimit for all labels when queue
+ // is initialized for the first time (when no applications are present).
+ for (String nodePartition : getNodeLabelsForQueue()) {
+ calculateAndGetAMResourceLimitPerPartition(nodePartition);
+ }
- for (Iterator<FiCaSchedulerApp> fsApp =
- getPendingAppsOrderingPolicy().getAssignmentIterator();
- fsApp.hasNext();) {
- FiCaSchedulerApp application = fsApp.next();
- ApplicationId applicationId = application.getApplicationId();
+ for (Iterator<FiCaSchedulerApp> fsApp =
+ getPendingAppsOrderingPolicy().getAssignmentIterator();
+ fsApp.hasNext(); ) {
+ FiCaSchedulerApp application = fsApp.next();
+ ApplicationId applicationId = application.getApplicationId();
- // Get the am-node-partition associated with each application
- // and calculate max-am resource limit for this partition.
- String partitionName = application.getAppAMNodePartitionName();
+ // Get the am-node-partition associated with each application
+ // and calculate max-am resource limit for this partition.
+ String partitionName = application.getAppAMNodePartitionName();
- Resource amLimit = getAMResourceLimitPerPartition(partitionName);
- // Verify whether we already calculated am-limit for this label.
- if (amLimit == null) {
- amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
- }
- // Check am resource limit.
- Resource amIfStarted = Resources.add(
- application.getAMResource(partitionName),
- queueUsage.getAMUsed(partitionName));
+ Resource amLimit = getAMResourceLimitPerPartition(partitionName);
+ // Verify whether we already calculated am-limit for this label.
+ if (amLimit == null) {
+ amLimit = calculateAndGetAMResourceLimitPerPartition(partitionName);
+ }
+ // Check am resource limit.
+ Resource amIfStarted = Resources.add(
+ application.getAMResource(partitionName),
+ queueUsage.getAMUsed(partitionName));
- if (LOG.isDebugEnabled()) {
- LOG.debug("application "+application.getId() +" AMResource "
- + application.getAMResource(partitionName)
- + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent
- + " amLimit " + amLimit + " lastClusterResource "
- + lastClusterResource + " amIfStarted " + amIfStarted
- + " AM node-partition name " + partitionName);
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("application " + application.getId() + " AMResource "
+ + application.getAMResource(partitionName)
+ + " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent
+ + " amLimit " + amLimit + " lastClusterResource "
+ + lastClusterResource + " amIfStarted " + amIfStarted
+ + " AM node-partition name " + partitionName);
+ }
- if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
- amIfStarted, amLimit)) {
- if (getNumActiveApplications() < 1
- || (Resources.lessThanOrEqual(resourceCalculator,
- lastClusterResource, queueUsage.getAMUsed(partitionName),
- Resources.none()))) {
- LOG.warn("maximum-am-resource-percent is insufficient to start a"
- + " single application in queue, it is likely set too low."
- + " skipping enforcement to allow at least one application"
- + " to start");
- } else {
- application.updateAMContainerDiagnostics(AMState.INACTIVATED,
- CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
- LOG.info("Not activating application " + applicationId
- + " as amIfStarted: " + amIfStarted + " exceeds amLimit: "
- + amLimit);
- continue;
+ if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
+ amIfStarted, amLimit)) {
+ if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(
+ resourceCalculator, lastClusterResource,
+ queueUsage.getAMUsed(partitionName), Resources.none()))) {
+ LOG.warn("maximum-am-resource-percent is insufficient to start a"
+ + " single application in queue, it is likely set too low."
+ + " skipping enforcement to allow at least one application"
+ + " to start");
+ } else{
+ application.updateAMContainerDiagnostics(AMState.INACTIVATED,
+ CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
+ LOG.info("Not activating application " + applicationId
+ + " as amIfStarted: " + amIfStarted + " exceeds amLimit: "
+ + amLimit);
+ continue;
+ }
}
- }
- // Check user am resource limit
- User user = getUser(application.getUser());
- Resource userAMLimit = userAmPartitionLimit.get(partitionName);
+ // Check user am resource limit
+ User user = getUser(application.getUser());
+ Resource userAMLimit = userAmPartitionLimit.get(partitionName);
- // Verify whether we already calculated user-am-limit for this label.
- if (userAMLimit == null) {
- userAMLimit = getUserAMResourceLimitPerPartition(partitionName);
- userAmPartitionLimit.put(partitionName, userAMLimit);
- }
+ // Verify whether we already calculated user-am-limit for this label.
+ if (userAMLimit == null) {
+ userAMLimit = getUserAMResourceLimitPerPartition(partitionName);
+ userAmPartitionLimit.put(partitionName, userAMLimit);
+ }
- Resource userAmIfStarted = Resources.add(
- application.getAMResource(partitionName),
- user.getConsumedAMResources(partitionName));
-
- if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
- userAmIfStarted, userAMLimit)) {
- if (getNumActiveApplications() < 1
- || (Resources.lessThanOrEqual(resourceCalculator,
- lastClusterResource, queueUsage.getAMUsed(partitionName),
- Resources.none()))) {
- LOG.warn("maximum-am-resource-percent is insufficient to start a"
- + " single application in queue for user, it is likely set too"
- + " low. skipping enforcement to allow at least one application"
- + " to start");
- } else {
- application.updateAMContainerDiagnostics(AMState.INACTIVATED,
- CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
- LOG.info("Not activating application " + applicationId
- + " for user: " + user + " as userAmIfStarted: "
- + userAmIfStarted + " exceeds userAmLimit: " + userAMLimit);
- continue;
+ Resource userAmIfStarted = Resources.add(
+ application.getAMResource(partitionName),
+ user.getConsumedAMResources(partitionName));
+
+ if (!Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
+ userAmIfStarted, userAMLimit)) {
+ if (getNumActiveApplications() < 1 || (Resources.lessThanOrEqual(
+ resourceCalculator, lastClusterResource,
+ queueUsage.getAMUsed(partitionName), Resources.none()))) {
+ LOG.warn("maximum-am-resource-percent is insufficient to start a"
+ + " single application in queue for user, it is likely set too"
+ + " low. skipping enforcement to allow at least one application"
+ + " to start");
+ } else{
+ application.updateAMContainerDiagnostics(AMState.INACTIVATED,
+ CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
+ LOG.info(
+ "Not activating application " + applicationId + " for user: "
+ + user + " as userAmIfStarted: " + userAmIfStarted
+ + " exceeds userAmLimit: " + userAMLimit);
+ continue;
+ }
}
+ user.activateApplication();
+ orderingPolicy.addSchedulableEntity(application);
+ application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
+
+ queueUsage.incAMUsed(partitionName,
+ application.getAMResource(partitionName));
+ user.getResourceUsage().incAMUsed(partitionName,
+ application.getAMResource(partitionName));
+ user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
+ metrics.incAMUsed(application.getUser(),
+ application.getAMResource(partitionName));
+ metrics.setAMResouceLimitForUser(application.getUser(), userAMLimit);
+ fsApp.remove();
+ LOG.info("Application " + applicationId + " from user: " + application
+ .getUser() + " activated in queue: " + getQueueName());
}
- user.activateApplication();
- orderingPolicy.addSchedulableEntity(application);
- application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
-
- queueUsage.incAMUsed(partitionName,
- application.getAMResource(partitionName));
- user.getResourceUsage().incAMUsed(partitionName,
- application.getAMResource(partitionName));
- user.getResourceUsage().setAMLimit(partitionName, userAMLimit);
- metrics.incAMUsed(application.getUser(),
- application.getAMResource(partitionName));
- metrics.setAMResouceLimitForUser(application.getUser(), userAMLimit);
- fsApp.remove();
- LOG.info("Application " + applicationId + " from user: "
- + application.getUser() + " activated in queue: " + getQueueName());
+ } finally {
+ writeLock.unlock();
}
}
- private synchronized void addApplicationAttempt(FiCaSchedulerApp application,
+ private void addApplicationAttempt(FiCaSchedulerApp application,
User user) {
- // Accept
- user.submitApplication();
- getPendingAppsOrderingPolicy().addSchedulableEntity(application);
- applicationAttemptMap.put(application.getApplicationAttemptId(), application);
+ try {
+ writeLock.lock();
+ // Accept
+ user.submitApplication();
+ getPendingAppsOrderingPolicy().addSchedulableEntity(application);
+ applicationAttemptMap.put(application.getApplicationAttemptId(),
+ application);
- // Activate applications
- activateApplications();
-
- LOG.info("Application added -" +
- " appId: " + application.getApplicationId() +
- " user: " + application.getUser() + "," +
- " leaf-queue: " + getQueueName() +
- " #user-pending-applications: " + user.getPendingApplications() +
- " #user-active-applications: " + user.getActiveApplications() +
- " #queue-pending-applications: " + getNumPendingApplications() +
- " #queue-active-applications: " + getNumActiveApplications()
- );
+ // Activate applications
+ activateApplications();
+
+ LOG.info(
+ "Application added -" + " appId: " + application.getApplicationId()
+ + " user: " + application.getUser() + "," + " leaf-queue: "
+ + getQueueName() + " #user-pending-applications: " + user
+ .getPendingApplications() + " #user-active-applications: " + user
+ .getActiveApplications() + " #queue-pending-applications: "
+ + getNumPendingApplications() + " #queue-active-applications: "
+ + getNumActiveApplications());
+ } finally {
+ writeLock.unlock();
+ }
}
@Override
@@ -767,49 +850,54 @@ public class LeafQueue extends AbstractCSQueue {
@Override
public void finishApplicationAttempt(FiCaSchedulerApp application, String queue) {
// Careful! Locking order is important!
- synchronized (this) {
- removeApplicationAttempt(application, getUser(application.getUser()));
- }
+ removeApplicationAttempt(application, application.getUser());
getParent().finishApplicationAttempt(application, queue);
}
- public synchronized void removeApplicationAttempt(
- FiCaSchedulerApp application, User user) {
- String partitionName = application.getAppAMNodePartitionName();
- boolean wasActive =
- orderingPolicy.removeSchedulableEntity(application);
- if (!wasActive) {
- pendingOrderingPolicy.removeSchedulableEntity(application);
- } else {
- queueUsage.decAMUsed(partitionName,
- application.getAMResource(partitionName));
- user.getResourceUsage().decAMUsed(partitionName,
- application.getAMResource(partitionName));
- metrics.decAMUsed(application.getUser(),
- application.getAMResource(partitionName));
- }
- applicationAttemptMap.remove(application.getApplicationAttemptId());
+ private void removeApplicationAttempt(
+ FiCaSchedulerApp application, String userName) {
+ try {
+ writeLock.lock();
- user.finishApplication(wasActive);
- if (user.getTotalApplications() == 0) {
- users.remove(application.getUser());
- }
+ // TODO, should use getUser, use this method just to avoid UT failure
+ // which is caused by wrong invoking order, will fix UT separately
+ User user = getUserAndAddIfAbsent(userName);
+
+ String partitionName = application.getAppAMNodePartitionName();
+ boolean wasActive = orderingPolicy.removeSchedulableEntity(application);
+ if (!wasActive) {
+ pendingOrderingPolicy.removeSchedulableEntity(application);
+ } else{
+ queueUsage.decAMUsed(partitionName,
+ application.getAMResource(partitionName));
+ user.getResourceUsage().decAMUsed(partitionName,
+ application.getAMResource(partitionName));
+ metrics.decAMUsed(application.getUser(),
+ application.getAMResource(partitionName));
+ }
+ applicationAttemptMap.remove(application.getApplicationAttemptId());
+
+ user.finishApplication(wasActive);
+ if (user.getTotalApplications() == 0) {
+ users.remove(application.getUser());
+ }
- // Check if we can activate more applications
- activateApplications();
+ // Check if we can activate more applications
+ activateApplications();
- LOG.info("Application removed -" +
- " appId: " + application.getApplicationId() +
- " user: " + application.getUser() +
- " queue: " + getQueueName() +
- " #user-pending-applications: " + user.getPendingApplications() +
- " #user-active-applications: " + user.getActiveApplications() +
- " #queue-pending-applications: " + getNumPendingApplications() +
- " #queue-active-applications: " + getNumActiveApplications()
- );
+ LOG.info(
+ "Application removed -" + " appId: " + application.getApplicationId()
+ + " user: " + application.getUser() + " queue: " + getQueueName()
+ + " #user-pending-applications: " + user.getPendingApplications()
+ + " #user-active-applications: " + user.getActiveApplications()
+ + " #queue-pending-applications: " + getNumPendingApplications()
+ + " #queue-active-applications: " + getNumActiveApplications());
+ } finally {
+ writeLock.unlock();
+ }
}
- private synchronized FiCaSchedulerApp getApplication(
+ private FiCaSchedulerApp getApplication(
ApplicationAttemptId applicationAttemptId) {
return applicationAttemptMap.get(applicationAttemptId);
}
@@ -871,170 +959,171 @@ public class LeafQueue extends AbstractCSQueue {
}
@Override
- public synchronized CSAssignment assignContainers(Resource clusterResource,
+ public CSAssignment assignContainers(Resource clusterResource,
FiCaSchedulerNode node, ResourceLimits currentResourceLimits,
SchedulingMode schedulingMode) {
- updateCurrentResourceLimits(currentResourceLimits, clusterResource);
+ try {
+ writeLock.lock();
+ updateCurrentResourceLimits(currentResourceLimits, clusterResource);
- if (LOG.isDebugEnabled()) {
- LOG.debug("assignContainers: node=" + node.getNodeName()
- + " #applications=" + orderingPolicy.getNumSchedulableEntities());
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ "assignContainers: node=" + node.getNodeName() + " #applications="
+ + orderingPolicy.getNumSchedulableEntities());
+ }
- setPreemptionAllowed(currentResourceLimits, node.getPartition());
+ setPreemptionAllowed(currentResourceLimits, node.getPartition());
- // Check for reserved resources
- RMContainer reservedContainer = node.getReservedContainer();
- if (reservedContainer != null) {
- FiCaSchedulerApp application =
- getApplication(reservedContainer.getApplicationAttemptId());
+ // Check for reserved resources
+ RMContainer reservedContainer = node.getReservedContainer();
+ if (reservedContainer != null) {
+ FiCaSchedulerApp application = getApplication(
+ reservedContainer.getApplicationAttemptId());
- ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
- node.getNodeID(), SystemClock.getInstance().getTime(), application);
+ ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
+ node.getNodeID(), SystemClock.getInstance().getTime(), application);
- synchronized (application) {
- CSAssignment assignment =
- application.assignContainers(clusterResource, node,
- currentResourceLimits, schedulingMode, reservedContainer);
+ CSAssignment assignment = application.assignContainers(clusterResource,
+ node, currentResourceLimits, schedulingMode, reservedContainer);
handleExcessReservedContainer(clusterResource, assignment, node,
application);
killToPreemptContainers(clusterResource, node, assignment);
return assignment;
}
- }
- // if our queue cannot access this node, just return
- if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
- && !accessibleToPartition(node.getPartition())) {
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParent().getQueueName(), getQueueName(), ActivityState.REJECTED,
- ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + node
- .getPartition());
- return CSAssignment.NULL_ASSIGNMENT;
- }
-
- // Check if this queue need more resource, simply skip allocation if this
- // queue doesn't need more resources.
- if (!hasPendingResourceRequest(node.getPartition(), clusterResource,
- schedulingMode)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Skip this queue=" + getQueuePath()
- + ", because it doesn't need more resource, schedulingMode="
- + schedulingMode.name() + " node-partition=" + node.getPartition());
+ // if our queue cannot access this node, just return
+ if (schedulingMode == SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY
+ && !accessibleToPartition(node.getPartition())) {
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParent().getQueueName(), getQueueName(), ActivityState.REJECTED,
+ ActivityDiagnosticConstant.NOT_ABLE_TO_ACCESS_PARTITION + node
+ .getPartition());
+ return CSAssignment.NULL_ASSIGNMENT;
}
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
- return CSAssignment.NULL_ASSIGNMENT;
- }
-
- for (Iterator<FiCaSchedulerApp> assignmentIterator =
- orderingPolicy.getAssignmentIterator(); assignmentIterator.hasNext();) {
- FiCaSchedulerApp application = assignmentIterator.next();
-
- ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
- node.getNodeID(), SystemClock.getInstance().getTime(), application);
- // Check queue max-capacity limit
- if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
- currentResourceLimits, application.getCurrentReservation(),
+ // Check if this queue need more resource, simply skip allocation if this
+ // queue doesn't need more resources.
+ if (!hasPendingResourceRequest(node.getPartition(), clusterResource,
schedulingMode)) {
- ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
- activitiesManager, node,
- application, application.getPriority(),
- ActivityDiagnosticConstant.QUEUE_MAX_CAPACITY_LIMIT);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Skip this queue=" + getQueuePath()
+ + ", because it doesn't need more resource, schedulingMode="
+ + schedulingMode.name() + " node-partition=" + node
+ .getPartition());
+ }
ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.EMPTY);
+ ActivityDiagnosticConstant.QUEUE_DO_NOT_NEED_MORE_RESOURCE);
return CSAssignment.NULL_ASSIGNMENT;
}
- Resource userLimit =
- computeUserLimitAndSetHeadroom(application, clusterResource,
- node.getPartition(), schedulingMode);
-
- // Check user limit
- if (!canAssignToUser(clusterResource, application.getUser(), userLimit,
- application, node.getPartition(), currentResourceLimits)) {
- application.updateAMContainerDiagnostics(AMState.ACTIVATED,
- "User capacity has reached its maximum limit.");
- ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
- activitiesManager, node,
- application, application.getPriority(),
- ActivityDiagnosticConstant.USER_CAPACITY_MAXIMUM_LIMIT);
- continue;
- }
+ for (Iterator<FiCaSchedulerApp> assignmentIterator =
+ orderingPolicy.getAssignmentIterator();
+ assignmentIterator.hasNext(); ) {
+ FiCaSchedulerApp application = assignmentIterator.next();
+
+ ActivitiesLogger.APP.startAppAllocationRecording(activitiesManager,
+ node.getNodeID(), SystemClock.getInstance().getTime(), application);
+
+ // Check queue max-capacity limit
+ if (!super.canAssignToThisQueue(clusterResource, node.getPartition(),
+ currentResourceLimits, application.getCurrentReservation(),
+ schedulingMode)) {
+ ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
+ activitiesManager, node, application, application.getPriority(),
+ ActivityDiagnosticConstant.QUEUE_MAX_CAPACITY_LIMIT);
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
+ ActivityDiagnosticConstant.EMPTY);
+ return CSAssignment.NULL_ASSIGNMENT;
+ }
- // Try to schedule
- CSAssignment assignment =
- application.assignContainers(clusterResource, node,
- currentResourceLimits, schedulingMode, null);
+ Resource userLimit = computeUserLimitAndSetHeadroom(application,
+ clusterResource, node.getPartition(), schedulingMode);
+
+ // Check user limit
+ if (!canAssignToUser(clusterResource, application.getUser(), userLimit,
+ application, node.getPartition(), currentResourceLimits)) {
+ application.updateAMContainerDiagnostics(AMState.ACTIVATED,
+ "User capacity has reached its maximum limit.");
+ ActivitiesLogger.APP.recordRejectedAppActivityFromLeafQueue(
+ activitiesManager, node, application, application.getPriority(),
+ ActivityDiagnosticConstant.USER_CAPACITY_MAXIMUM_LIMIT);
+ continue;
+ }
- if (LOG.isDebugEnabled()) {
- LOG.debug("post-assignContainers for application "
- + application.getApplicationId());
- application.showRequests();
- }
+ // Try to schedule
+ CSAssignment assignment = application.assignContainers(clusterResource,
+ node, currentResourceLimits, schedulingMode, null);
- // Did we schedule or reserve a container?
- Resource assigned = assignment.getResource();
-
- handleExcessReservedContainer(clusterResource, assignment, node,
- application);
- killToPreemptContainers(clusterResource, node, assignment);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("post-assignContainers for application " + application
+ .getApplicationId());
+ application.showRequests();
+ }
- if (Resources.greaterThan(resourceCalculator, clusterResource, assigned,
- Resources.none())) {
- // Get reserved or allocated container from application
- RMContainer reservedOrAllocatedRMContainer =
- application.getRMContainer(assignment.getAssignmentInformation()
- .getFirstAllocatedOrReservedContainerId());
+ // Did we schedule or reserve a container?
+ Resource assigned = assignment.getResource();
- // Book-keeping
- // Note: Update headroom to account for current allocation too...
- allocateResource(clusterResource, application, assigned,
- node.getPartition(), reservedOrAllocatedRMContainer,
- assignment.isIncreasedAllocation());
-
- // Update reserved metrics
- Resource reservedRes = assignment.getAssignmentInformation()
- .getReserved();
- if (reservedRes != null && !reservedRes.equals(Resources.none())) {
- incReservedResource(node.getPartition(), reservedRes);
- }
+ handleExcessReservedContainer(clusterResource, assignment, node,
+ application);
+ killToPreemptContainers(clusterResource, node, assignment);
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParent().getQueueName(), getQueueName(), ActivityState.ACCEPTED,
- ActivityDiagnosticConstant.EMPTY);
+ if (Resources.greaterThan(resourceCalculator, clusterResource, assigned,
+ Resources.none())) {
+ // Get reserved or allocated container from application
+ RMContainer reservedOrAllocatedRMContainer =
+ application.getRMContainer(assignment.getAssignmentInformation()
+ .getFirstAllocatedOrReservedContainerId());
+
+ // Book-keeping
+ // Note: Update headroom to account for current allocation too...
+ allocateResource(clusterResource, application, assigned,
+ node.getPartition(), reservedOrAllocatedRMContainer,
+ assignment.isIncreasedAllocation());
+
+ // Update reserved metrics
+ Resource reservedRes =
+ assignment.getAssignmentInformation().getReserved();
+ if (reservedRes != null && !reservedRes.equals(Resources.none())) {
+ incReservedResource(node.getPartition(), reservedRes);
+ }
- // Done
- return assignment;
- } else if (assignment.getSkippedType()
- == CSAssignment.SkippedType.OTHER) {
- ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
- activitiesManager, application.getApplicationId(),
- ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
- application.updateNodeInfoForAMDiagnostics(node);
- } else if(assignment.getSkippedType()
- == CSAssignment.SkippedType.QUEUE_LIMIT) {
- return assignment;
- } else {
- // If we don't allocate anything, and it is not skipped by application,
- // we will return to respect FIFO of applications
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.RESPECT_FIFO);
- ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
- activitiesManager, application.getApplicationId(),
- ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
- return CSAssignment.NULL_ASSIGNMENT;
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParent().getQueueName(), getQueueName(),
+ ActivityState.ACCEPTED, ActivityDiagnosticConstant.EMPTY);
+
+ // Done
+ return assignment;
+ } else if (assignment.getSkippedType()
+ == CSAssignment.SkippedType.OTHER) {
+ ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
+ activitiesManager, application.getApplicationId(),
+ ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
+ application.updateNodeInfoForAMDiagnostics(node);
+ } else if (assignment.getSkippedType()
+ == CSAssignment.SkippedType.QUEUE_LIMIT) {
+ return assignment;
+ } else{
+ // If we don't allocate anything, and it is not skipped by application,
+ // we will return to respect FIFO of applications
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
+ ActivityDiagnosticConstant.RESPECT_FIFO);
+ ActivitiesLogger.APP.finishSkippedAppAllocationRecording(
+ activitiesManager, application.getApplicationId(),
+ ActivityState.SKIPPED, ActivityDiagnosticConstant.EMPTY);
+ return CSAssignment.NULL_ASSIGNMENT;
+ }
}
- }
- ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
- getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
- ActivityDiagnosticConstant.EMPTY);
+ ActivitiesLogger.QUEUE.recordQueueActivity(activitiesManager, node,
+ getParent().getQueueName(), getQueueName(), ActivityState.SKIPPED,
+ ActivityDiagnosticConstant.EMPTY);
- return CSAssignment.NULL_ASSIGNMENT;
+ return CSAssignment.NULL_ASSIGNMENT;
+ } finally {
+ writeLock.unlock();
+ }
}
protected Resource getHeadroom(User user, Resource queueCurrentLimit,
@@ -1109,7 +1198,8 @@ public class LeafQueue extends AbstractCSQueue {
}
}
- @Lock({LeafQueue.class, FiCaSchedulerApp.class})
+ // It doesn't necessarily to hold application's lock here.
+ @Lock({LeafQueue.class})
Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
Resource clusterResource, String nodePartition,
SchedulingMode schedulingMode) {
@@ -1281,51 +1371,53 @@ public class LeafQueue extends AbstractCSQueue {
}
@Private
- protected synchronized boolean canAssignToUser(Resource clusterResource,
+ protected boolean canAssignToUser(Resource clusterResource,
String userName, Resource limit, FiCaSchedulerApp application,
String nodePartition, ResourceLimits currentResourceLimits) {
- User user = getUser(userName);
-
- currentResourceLimits.setAmountNeededUnreserve(Resources.none());
-
- // Note: We aren't considering the current request since there is a fixed
- // overhead of the AM, but it's a > check, not a >= check, so...
- if (Resources
- .greaterThan(resourceCalculator, clusterResource,
- user.getUsed(nodePartition),
- limit)) {
- // if enabled, check to see if could we potentially use this node instead
- // of a reserved node if the application has reserved containers
- if (this.reservationsContinueLooking &&
- nodePartition.equals(CommonNodeLabelsManager.NO_LABEL)) {
- if (Resources.lessThanOrEqual(
- resourceCalculator,
- clusterResource,
- Resources.subtract(user.getUsed(),
- application.getCurrentReservation()), limit)) {
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("User " + userName + " in queue " + getQueueName()
- + " will exceed limit based on reservations - " + " consumed: "
- + user.getUsed() + " reserved: "
- + application.getCurrentReservation() + " limit: " + limit);
+ try {
+ readLock.lock();
+ User user = getUser(userName);
+
+ currentResourceLimits.setAmountNeededUnreserve(Resources.none());
+
+ // Note: We aren't considering the current request since there is a fixed
+ // overhead of the AM, but it's a > check, not a >= check, so...
+ if (Resources.greaterThan(resourceCalculator, clusterResource,
+ user.getUsed(nodePartition), limit)) {
+ // if enabled, check to see if could we potentially use this node instead
+ // of a reserved node if the application has reserved containers
+ if (this.reservationsContinueLooking && nodePartition.equals(
+ CommonNodeLabelsManager.NO_LABEL)) {
+ if (Resources.lessThanOrEqual(resourceCalculator, clusterResource,
+ Resources.subtract(user.getUsed(),
+ application.getCurrentReservation()), limit)) {
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User " + userName + " in queue " + getQueueName()
+ + " will exceed limit based on reservations - "
+ + " consumed: " + user.getUsed() + " reserved: " + application
+ .getCurrentReservation() + " limit: " + limit);
+ }
+ Resource amountNeededToUnreserve = Resources.subtract(
+ user.getUsed(nodePartition), limit);
+ // we can only acquire a new container if we unreserve first to
+ // respect user-limit
+ currentResourceLimits.setAmountNeededUnreserve(
+ amountNeededToUnreserve);
+ return true;
}
- Resource amountNeededToUnreserve =
- Resources.subtract(user.getUsed(nodePartition), limit);
- // we can only acquire a new container if we unreserve first to
- // respect user-limit
- currentResourceLimits.setAmountNeededUnreserve(amountNeededToUnreserve);
- return true;
}
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("User " + userName + " in queue " + getQueueName()
+ + " will exceed limit - " + " consumed: " + user
+ .getUsed(nodePartition) + " limit: " + limit);
+ }
+ return false;
}
- if (LOG.isDebugEnabled()) {
- LOG.debug("User " + userName + " in queue " + getQueueName()
- + " will exceed limit - " + " consumed: "
- + user.getUsed(nodePartition) + " limit: " + limit);
- }
- return false;
+ return true;
+ } finally {
+ readLock.unlock();
}
- return true;
}
@Override
@@ -1333,15 +1425,15 @@ public class LeafQueue extends AbstractCSQueue {
FiCaSchedulerApp app, FiCaSchedulerNode node, RMContainer rmContainer) {
boolean removed = false;
Priority priority = null;
-
- synchronized (this) {
+
+ try {
+ writeLock.lock();
if (rmContainer.getContainer() != null) {
priority = rmContainer.getContainer().getPriority();
}
if (null != priority) {
- removed = app.unreserve(
- rmContainer.getAllocatedSchedulerKey(), node,
+ removed = app.unreserve(rmContainer.getAllocatedSchedulerKey(), node,
rmContainer);
}
@@ -1352,8 +1444,10 @@ public class LeafQueue extends AbstractCSQueue {
releaseResource(clusterResource, app, rmContainer.getReservedResource(),
node.getPartition(), rmContainer, true);
}
+ } finally {
+ writeLock.unlock();
}
-
+
if (removed) {
getParent().unreserveIncreasedContainer(clusterResource, app, node,
rmContainer);
@@ -1380,42 +1474,52 @@ public class LeafQueue extends AbstractCSQueue {
}
}
- private synchronized float calculateUserUsageRatio(Resource clusterResource,
+ private float calculateUserUsageRatio(Resource clusterResource,
String nodePartition) {
- Resource resourceByLabel =
- labelManager.getResourceByLabel(nodePartition, clusterResource);
- float consumed = 0;
- User user;
- for (Map.Entry<String, User> entry : users.entrySet()) {
- user = entry.getValue();
- consumed += user.resetAndUpdateUsageRatio(resourceCalculator,
- resourceByLabel, nodePartition);
+ try {
+ writeLock.lock();
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
+ clusterResource);
+ float consumed = 0;
+ User user;
+ for (Map.Entry<String, User> entry : users.entrySet()) {
+ user = entry.getValue();
+ consumed += user.resetAndUpdateUsageRatio(resourceCalculator,
+ resourceByLabel, nodePartition);
+ }
+ return consumed;
+ } finally {
+ writeLock.unlock();
}
- return consumed;
}
- private synchronized void recalculateQueueUsageRatio(Resource clusterResource,
+ private void recalculateQueueUsageRatio(Resource clusterResource,
String nodePartition) {
- ResourceUsage queueResourceUsage = this.getQueueResourceUsage();
-
- if (nodePartition == null) {
- for (String partition : Sets.union(queueCapacities.getNodePartitionsSet(),
- queueResourceUsage.getNodePartitionsSet())) {
- qUsageRatios.setUsageRatio(partition,
- calculateUserUsageRatio(clusterResource, partition));
+ try {
+ writeLock.lock();
+ ResourceUsage queueResourceUsage = this.getQueueResourceUsage();
+
+ if (nodePartition == null) {
+ for (String partition : Sets.union(
+ queueCapacities.getNodePartitionsSet(),
+ queueResourceUsage.getNodePartitionsSet())) {
+ qUsageRatios.setUsageRatio(partition,
+ calculateUserUsageRatio(clusterResource, partition));
+ }
+ } else{
+ qUsageRatios.setUsageRatio(nodePartition,
+ calculateUserUsageRatio(clusterResource, nodePartition));
}
- } else {
- qUsageRatios.setUsageRatio(nodePartition,
- calculateUserUsageRatio(clusterResource, nodePartition));
+ } finally {
+ writeLock.unlock();
}
}
- private synchronized void updateQueueUsageRatio(String nodePartition,
+ private void updateQueueUsageRatio(String nodePartition,
float delta) {
qUsageRatios.incUsageRatio(nodePartition, delta);
}
-
@Override
public void completedContainer(Resource clusterResource,
FiCaSchedulerApp application, FiCaSchedulerNode node, RMContainer rmContainer,
@@ -1438,21 +1542,20 @@ public class LeafQueue extends AbstractCSQueue {
boolean removed = false;
// Careful! Locking order is important!
- synchronized (this) {
-
+ try {
+ writeLock.lock();
Container container = rmContainer.getContainer();
// Inform the application & the node
// Note: It's safe to assume that all state changes to RMContainer
- // happen under scheduler's lock...
+ // happen under scheduler's lock...
// So, this is, in effect, a transaction across application & node
if (rmContainer.getState() == RMContainerState.RESERVED) {
removed = application.unreserve(rmContainer.getReservedSchedulerKey(),
node, rmContainer);
- } else {
- removed =
- application.containerCompleted(rmContainer, containerStatus,
- event, node.getPartition());
+ } else{
+ removed = application.containerCompleted(rmContainer, containerStatus,
+ event, node.getPartition());
node.releaseContainer(container);
}
@@ -1462,12 +1565,15 @@ public class LeafQueue extends AbstractCSQueue {
// Inform the ordering policy
orderingPolicy.containerReleased(application, rmContainer);
-
+
releaseResource(clusterResource, application, container.getResource(),
node.getPartition(), rmContainer, false);
}
+ } finally {
+ writeLock.unlock();
}
+
if (removed) {
// Inform the parent queue _outside_ of the leaf-queue lock
getParent().completedContainer(clusterResource, application, node,
@@ -1480,91 +1586,104 @@ public class LeafQueue extends AbstractCSQueue {
new KillableContainer(rmContainer, node.getPartition(), queueName));
}
- synchronized void allocateResource(Resource clusterResource,
+ void allocateResource(Resource clusterResource,
SchedulerApplicationAttempt application, Resource resource,
String nodePartition, RMContainer rmContainer,
boolean isIncreasedAllocation) {
- super.allocateResource(clusterResource, resource, nodePartition,
- isIncreasedAllocation);
- Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
- clusterResource);
-
- // handle ignore exclusivity container
- if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
- RMNodeLabelsManager.NO_LABEL)
- && !nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
- TreeSet<RMContainer> rmContainers = null;
- if (null == (rmContainers =
- ignorePartitionExclusivityRMContainers.get(nodePartition))) {
- rmContainers = new TreeSet<>();
- ignorePartitionExclusivityRMContainers.put(nodePartition, rmContainers);
+ try {
+ writeLock.lock();
+ super.allocateResource(clusterResource, resource, nodePartition,
+ isIncreasedAllocation);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
+ clusterResource);
+
+ // handle ignore exclusivity container
+ if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
+ RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
+ RMNodeLabelsManager.NO_LABEL)) {
+ TreeSet<RMContainer> rmContainers = null;
+ if (null == (rmContainers = ignorePartitionExclusivityRMContainers.get(
+ nodePartition))) {
+ rmContainers = new TreeSet<>();
+ ignorePartitionExclusivityRMContainers.put(nodePartition,
+ rmContainers);
+ }
+ rmContainers.add(rmContainer);
}
- rmContainers.add(rmContainer);
- }
- // Update user metrics
- String userName = application.getUser();
- User user = getUser(userName);
- user.assignContainer(resource, nodePartition);
+ // Update user metrics
+ String userName = application.getUser();
- // Update usage ratios
- updateQueueUsageRatio(nodePartition,
- user.updateUsageRatio(resourceCalculator, resourceByLabel,
- nodePartition));
+ // TODO, should use getUser, use this method just to avoid UT failure
+ // which is caused by wrong invoking order, will fix UT separately
+ User user = getUserAndAddIfAbsent(userName);
- // Note this is a bit unconventional since it gets the object and modifies
- // it here, rather then using set routine
- Resources.subtractFrom(application.getHeadroom(), resource); // headroom
- metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
-
- if (LOG.isDebugEnabled()) {
- LOG.debug(getQueueName() +
- " user=" + userName +
- " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
- " headroom = " + application.getHeadroom() +
- " user-resources=" + user.getUsed()
- );
+ user.assignContainer(resource, nodePartition);
+
+ // Update usage ratios
+ updateQueueUsageRatio(nodePartition,
+ user.updateUsageRatio(resourceCalculator, resourceByLabel,
+ nodePartition));
+
+ // Note this is a bit unconventional since it gets the object and modifies
+ // it here, rather then using set routine
+ Resources.subtractFrom(application.getHeadroom(), resource); // headroom
+ metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(getQueueName() + " user=" + userName + " used=" + queueUsage
+ .getUsed() + " numContainers=" + numContainers + " headroom = "
+ + application.getHeadroom() + " user-resources=" + user.getUsed());
+ }
+ } finally {
+ writeLock.unlock();
}
}
- synchronized void releaseResource(Resource clusterResource,
+ void releaseResource(Resource clusterResource,
FiCaSchedulerApp application, Resource resource, String nodePartition,
RMContainer rmContainer, boolean isChangeResource) {
- super.releaseResource(clusterResource, resource, nodePartition,
- isChangeResource);
- Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
- clusterResource);
-
- // handle ignore exclusivity container
- if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
- RMNodeLabelsManager.NO_LABEL)
- && !nodePartition.equals(RMNodeLabelsManager.NO_LABEL)) {
- if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) {
- Set<RMContainer> rmContainers =
- ignorePartitionExclusivityRMContainers.get(nodePartition);
- rmContainers.remove(rmContainer);
- if (rmContainers.isEmpty()) {
- ignorePartitionExclusivityRMContainers.remove(nodePartition);
+ try {
+ writeLock.lock();
+ super.releaseResource(clusterResource, resource, nodePartition,
+ isChangeResource);
+ Resource resourceByLabel = labelManager.getResourceByLabel(nodePartition,
+ clusterResource);
+
+ // handle ignore exclusivity container
+ if (null != rmContainer && rmContainer.getNodeLabelExpression().equals(
+ RMNodeLabelsManager.NO_LABEL) && !nodePartition.equals(
+ RMNodeLabelsManager.NO_LABEL)) {
+ if (ignorePartitionExclusivityRMContainers.containsKey(nodePartition)) {
+ Set<RMContainer> rmContainers =
+ ignorePartitionExclusivityRMContainers.get(nodePartition);
+ rmContainers.remove(rmContainer);
+ if (rmContainers.isEmpty()) {
+ ignorePartitionExclusivityRMContainers.remove(nodePartition);
+ }
}
}
- }
- // Update user metrics
- String userName = application.getUser();
- User user = getUser(userName);
- user.releaseContainer(resource, nodePartition);
+ // Update user metrics
+ String userName = application.getUser();
+ User user = getUserAndAddIfAbsent(userName);
+ user.releaseContainer(resource, nodePartition);
- // Update usage ratios
- updateQueueUsageRatio(nodePartition,
- user.updateUsageRatio(resourceCalculator, resourceByLabel,
- nodePartition));
+ // Update usage ratios
+ updateQueueUsageRatio(nodePartition,
+ user.updateUsageRatio(resourceCalculator, resourceByLabel,
+ nodePartition));
- metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
+ metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
- if (LOG.isDebugEnabled()) {
- LOG.debug(getQueueName() +
- " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
- " user=" + userName + " user-resources=" + user.getUsed());
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(
+ getQueueName() + " used=" + queueUsage.getUsed() + " numContainers="
+ + numContainers + " user=" + userName + " user-resources="
+ + user.getUsed());
+ }
+ } finally {
+ writeLock.unlock();
}
}
@@ -1589,35 +1708,38 @@ public class LeafQueue extends AbstractCSQueue {
}
@Override
- public synchronized void updateClusterResource(Resource clusterResource,
+ public void updateClusterResource(Resource clusterResource,
ResourceLimits currentResourceLimits) {
- updateCurrentResourceLimits(currentResourceLimits, clusterResource);
- lastClusterResource = clusterResource;
-
- // Update headroom info based on new cluster resource value
- // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
- // during allocation
- setQueueResourceLimitsInfo(clusterResource);
+ try {
+ writeLock.lock();
+ updateCurrentResourceLimits(currentResourceLimits, clusterResource);
+ lastClusterResource = clusterResource;
- // Update user consumedRatios
- recalculateQueueUsageRatio(clusterResource, null);
+ // Update headroom info based on new cluster resource value
+ // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
+ // during allocation
+ setQueueResourceLimitsInfo(clusterResource);
- // Update metrics
- CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
- minimumAllocation, this, labelManager, null);
+ // Update user consumedRatios
+ recalculateQueueUsageRatio(clusterResource, null);
- // queue metrics are updated, more resource may be available
- // activate the pending applications if possible
- activateApplications();
+ // Update metrics
+ CSQueueUtils.updateQueueStatistics(resourceCalculator, clusterResource,
+ minimumAllocation, this, labelManager, null);
- // Update application properties
- for (FiCaSchedulerApp application :
- orderingPolicy.getSchedulableEntities()) {
- synchronized (application) {
+ // queue metrics are updated, more resource may be available
+ // activate the pending applications if possible
+ activateApplications();
+
+ // Update application properties
+ for (FiCaSchedulerApp application : orderingPolicy
+ .getSchedulableEntities()) {
computeUserLimitAndSetHeadroom(application, clusterResource,
RMNodeLabelsManager.NO_LABEL,
SchedulingMode.RESPECT_PARTITION_EXCLUSIVITY);
}
+ } finally {
+ writeLock.unlock();
}
}
@@ -1714,30 +1836,47 @@ public class LeafQueue extends AbstractCSQueue {
public static class User {
ResourceUsage userResourceUsage = new ResourceUsage();
volatile Resource userResourceLimit = Resource.newInstance(0, 0);
- int pendingApplications = 0;
- int activeApplications = 0;
+ volatile int pendingApplications = 0;
+ volatile int activeApplications = 0;
private UsageRatios userUsageRatios = new UsageRatios();
+ private WriteLock writeLock;
+
+ User() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ // Nobody uses read-lock now, will add it when necessary
+ writeLock = lock.writeLock();
+ }
public ResourceUsage getResourceUsage() {
return userResourceUsage;
}
- public synchronized float resetAndUpdateUsageRatio(
+ public float resetAndUpdateUsageRatio(
ResourceCalculator resourceCalculator,
Resource resource, String nodePartition) {
- userUsageRatios.setUsageRatio(nodePartition, 0);
- return updateUsageRatio(resourceCalculator, resource, nodePartition);
+ try {
+ writeLock.lock();
+ userUsageRatios.setUsageRatio(nodePartition, 0);
+ return updateUsageRatio(resourceCalculator, resource, nodePartition);
+ } finally {
+ writeLock.unlock();
+ }
}
- public synchronized float updateUsageRatio(
+ public float updateUsageRatio(
ResourceCalculator resourceCalculator,
Resource resource, String nodePartition) {
- float delta;
- float newRatio =
- Resources.ratio(resourceCalculator, getUsed(nodePartition), resource);
- delta = newRatio - userUsageRatios.getUsageRatio(nodePartition);
- userUsageRatios.setUsageRatio(nodePartition, newRatio);
- return delta;
+ try {
+ writeLock.lock();
+ float delta;
+ float newRatio = Resources.ratio(resourceCalcula
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: TimelineClient failed to retry on
java.net.SocketTimeoutException: Read timed out
Posted by ji...@apache.org.
TimelineClient failed to retry on java.net.SocketTimeoutException: Read timed out
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e6ee957
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e6ee957
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e6ee957
Branch: refs/heads/MAPREDUCE-6608
Commit: 2e6ee957161ab63a02a7861b727efa6310b275b2
Parents: d85d9b2
Author: Varun Saxena <va...@apache.org>
Authored: Fri Sep 23 13:23:54 2016 +0530
Committer: Varun Saxena <va...@apache.org>
Committed: Fri Sep 23 13:23:54 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e6ee957/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 56da2a7..dc4d3e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -265,7 +265,8 @@ public class TimelineClientImpl extends TimelineClient {
public boolean shouldRetryOn(Exception e) {
// Only retry on connection exceptions
return (e instanceof ClientHandlerException)
- && (e.getCause() instanceof ConnectException);
+ && (e.getCause() instanceof ConnectException ||
+ e.getCause() instanceof SocketTimeoutException);
}
};
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-3877.
YarnClientImpl.submitApplication swallows exceptions. Contributed by Varun
Saxena
Posted by ji...@apache.org.
YARN-3877. YarnClientImpl.submitApplication swallows exceptions. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4e72db5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4e72db5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4e72db5
Branch: refs/heads/MAPREDUCE-6608
Commit: e4e72db5f9f305b493138ab36f073fe5d1750ad8
Parents: 3e37e24
Author: Naganarasimha <na...@apache.org>
Authored: Sun Sep 25 17:36:30 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sun Sep 25 17:36:30 2016 +0530
----------------------------------------------------------------------
.../yarn/client/api/impl/YarnClientImpl.java | 13 +++--
.../yarn/client/api/impl/TestYarnClient.java | 52 ++++++++++++++++++++
2 files changed, 60 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4e72db5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 7760521..80e453f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -306,9 +306,10 @@ public class YarnClientImpl extends YarnClient {
try {
Thread.sleep(submitPollIntervalMillis);
} catch (InterruptedException ie) {
- LOG.error("Interrupted while waiting for application "
- + applicationId
- + " to be successfully submitted.");
+ String msg = "Interrupted while waiting for application "
+ + applicationId + " to be successfully submitted.";
+ LOG.error(msg);
+ throw new YarnException(msg, ie);
}
} catch (ApplicationNotFoundException ex) {
// FailOver or RM restart happens before RMStateStore saves
@@ -446,8 +447,10 @@ public class YarnClientImpl extends YarnClient {
Thread.sleep(asyncApiPollIntervalMillis);
}
} catch (InterruptedException e) {
- LOG.error("Interrupted while waiting for application " + applicationId
- + " to be killed.");
+ String msg = "Interrupted while waiting for application "
+ + applicationId + " to be killed.";
+ LOG.error(msg);
+ throw new YarnException(msg, e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4e72db5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index e462be1..19966ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -27,6 +27,7 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.lang.Thread.State;
import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
@@ -201,6 +202,57 @@ public class TestYarnClient {
client.stop();
}
+ @SuppressWarnings("deprecation")
+ @Test (timeout = 20000)
+ public void testSubmitApplicationInterrupted() throws IOException {
+ Configuration conf = new Configuration();
+ int pollIntervalMs = 1000;
+ conf.setLong(YarnConfiguration.YARN_CLIENT_APP_SUBMISSION_POLL_INTERVAL_MS,
+ pollIntervalMs);
+ try (final YarnClient client = new MockYarnClient()) {
+ client.init(conf);
+ client.start();
+ // Submit the application and then interrupt it while its waiting
+ // for submission to be successful.
+ final class SubmitThread extends Thread {
+ private boolean isInterrupted = false;
+ @Override
+ public void run() {
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ApplicationId applicationId = ApplicationId.newInstance(
+ System.currentTimeMillis(), 1);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ ((MockYarnClient) client).setYarnApplicationState(
+ YarnApplicationState.NEW);
+ try {
+ client.submitApplication(context);
+ } catch (YarnException | IOException e) {
+ if (e instanceof YarnException && e.getCause() != null &&
+ e.getCause() instanceof InterruptedException) {
+ isInterrupted = true;
+ }
+ }
+ }
+ }
+ SubmitThread appSubmitThread = new SubmitThread();
+ appSubmitThread.start();
+ try {
+ // Wait for thread to start and begin to sleep
+ // (enter TIMED_WAITING state).
+ while (appSubmitThread.getState() != State.TIMED_WAITING) {
+ Thread.sleep(pollIntervalMs / 2);
+ }
+ // Interrupt the thread.
+ appSubmitThread.interrupt();
+ appSubmitThread.join();
+ } catch (InterruptedException e) {
+ }
+ Assert.assertTrue("Expected an InterruptedException wrapped inside a " +
+ "YarnException", appSubmitThread.isInterrupted);
+ }
+ }
+
@Test (timeout = 30000)
public void testSubmitIncorrectQueueToCapacityScheduler() throws IOException {
MiniYARNCluster cluster = new MiniYARNCluster("testMRAMTokens", 1, 1, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: HDFS-10868. Remove stray
references to DFS_HDFS_BLOCKS_METADATA_ENABLED.
Posted by ji...@apache.org.
HDFS-10868. Remove stray references to DFS_HDFS_BLOCKS_METADATA_ENABLED.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c54f6ef3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c54f6ef3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c54f6ef3
Branch: refs/heads/MAPREDUCE-6608
Commit: c54f6ef30fbd5fbb9663e182b76bafb55ef567ad
Parents: b8a30f2
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Sep 19 11:17:03 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Sep 19 11:17:03 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java | 3 ---
.../src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 ----
.../test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java | 2 --
3 files changed, 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c54f6ef3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
index 642d4c8..4c754d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsClientConfigKeys.java
@@ -131,9 +131,6 @@ public interface HdfsClientConfigKeys {
"dfs.client.key.provider.cache.expiry";
long DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
TimeUnit.DAYS.toMillis(10); // 10 days
- String DFS_HDFS_BLOCKS_METADATA_ENABLED =
- "dfs.datanode.hdfs-blocks-metadata.enabled";
- boolean DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT = false;
String DFS_DATANODE_KERBEROS_PRINCIPAL_KEY =
"dfs.datanode.kerberos.principal";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c54f6ef3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 3532d25..df45e2a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -58,10 +58,6 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
.DFS_CHECKSUM_TYPE_KEY;
public static final String DFS_CHECKSUM_TYPE_DEFAULT =
HdfsClientConfigKeys.DFS_CHECKSUM_TYPE_DEFAULT;
- public static final String DFS_HDFS_BLOCKS_METADATA_ENABLED =
- HdfsClientConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED;
- public static final boolean DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT =
- HdfsClientConfigKeys.DFS_HDFS_BLOCKS_METADATA_ENABLED_DEFAULT;
public static final String DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT =
HdfsClientConfigKeys.DFS_WEBHDFS_ACL_PERMISSION_PATTERN_DEFAULT;
public static final String DFS_WEBHDFS_NETTY_LOW_WATERMARK =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c54f6ef3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
index 46420f1..bf29428 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/tools/TestHdfsConfigFields.java
@@ -76,8 +76,6 @@ public class TestHdfsConfigFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add("dfs.corruptfilesreturned.max");
configurationPropsToSkipCompare
- .add("dfs.datanode.hdfs-blocks-metadata.enabled");
- configurationPropsToSkipCompare
.add("dfs.metrics.session-id");
configurationPropsToSkipCompare
.add("dfs.datanode.synconclose");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: MAPREDUCE-6777. Typos in 4 log
messages. Contributed by Mehran Hassani
Posted by ji...@apache.org.
MAPREDUCE-6777. Typos in 4 log messages. Contributed by Mehran Hassani
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d21c280
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d21c280
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d21c280
Branch: refs/heads/MAPREDUCE-6608
Commit: 7d21c280a82b2f02675bf0048f0e965d99a05ae7
Parents: ade7c2b
Author: Naganarasimha <na...@apache.org>
Authored: Sat Sep 17 10:19:59 2016 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Sat Sep 17 10:19:59 2016 +0530
----------------------------------------------------------------------
.../java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java | 2 +-
.../apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java | 2 +-
.../src/main/java/org/apache/hadoop/mapred/Task.java | 2 +-
.../src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d21c280/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
index 49a00c5..2c0ea2b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/TaskAttemptListenerImpl.java
@@ -260,7 +260,7 @@ public class TaskAttemptListenerImpl extends CompositeService
@Override
public void done(TaskAttemptID taskAttemptID) throws IOException {
- LOG.info("Done acknowledgement from " + taskAttemptID.toString());
+ LOG.info("Done acknowledgment from " + taskAttemptID.toString());
org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID =
TypeConverter.toYarn(taskAttemptID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d21c280/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 53d8a36..e61bbaa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -400,7 +400,7 @@ public class JobHistoryEventHandler extends AbstractService
}
mi.shutDownTimer();
} catch (IOException e) {
- LOG.info("Exception while cancelling delayed flush timer. "
+ LOG.info("Exception while canceling delayed flush timer. "
+ "Likely caused by a failed flush " + e.getMessage());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d21c280/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
index 3c58a67..d20e9bd 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Task.java
@@ -1312,7 +1312,7 @@ abstract public class Task implements Writable, Configurable {
setPhase(TaskStatus.Phase.CLEANUP);
getProgress().setStatus("cleanup");
statusUpdate(umbilical);
- LOG.info("Runnning cleanup for the task");
+ LOG.info("Running cleanup for the task");
// do the cleanup
committer.abortTask(taskContext);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d21c280/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
index 7818c81..558ee38 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
@@ -672,7 +672,7 @@ public class ShuffleHandler extends AuxiliaryService {
return;
}
if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
- LOG.info("Storing state DB schedma version info " + getCurrentVersion());
+ LOG.info("Storing state DB schema version info " + getCurrentVersion());
storeVersion();
} else {
throw new IOException(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org