You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by na...@apache.org on 2018/04/02 07:18:17 UTC
[01/50] [abbrv] hadoop git commit: YARN-8062. yarn rmadmin -getGroups
returns group from which the user has been removed. (Sunil G via wangda)
[Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-3409 0df5c70e1 -> c20126acf (forced update)
YARN-8062. yarn rmadmin -getGroups returns group from which the user has been removed. (Sunil G via wangda)
Change-Id: I80ed63846502bf7751b890b6c6c6a7c0679e2b4a
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5d381570
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5d381570
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5d381570
Branch: refs/heads/YARN-3409
Commit: 5d381570f83022b411a8740d58486a7f68ab2af6
Parents: 22194f3
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Mar 26 11:16:06 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Mar 26 11:16:06 2018 -0700
----------------------------------------------------------------------
.../server/resourcemanager/ResourceManager.java | 20 +++++++++++++++++---
1 file changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5d381570/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 5140c9f..38da7f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -260,9 +260,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
// load core-site.xml
loadConfigurationXml(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
- // Do refreshUserToGroupsMappings with loaded core-site.xml
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
- .refresh();
+ // Refresh user to group mappings during init.
+ refreshUserToGroupMappingsWithConf();
// Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
// Or use RM specific configurations to overwrite the common ones first
@@ -347,6 +346,21 @@ public class ResourceManager extends CompositeService implements Recoverable {
super.serviceInit(this.conf);
}
+ private void refreshUserToGroupMappingsWithConf()
+ throws YarnException, IOException {
+ Configuration newConf = new Configuration(false);
+ InputStream confFileInputStream =
+ configurationProvider
+ .getConfigurationInputStream(newConf, YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
+ if (confFileInputStream != null) {
+ newConf.addResource(confFileInputStream);
+ }
+
+ // Do refreshUserToGroupsMappings with loaded core-site.xml
+ Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(newConf)
+ .refresh();
+ }
+
private void loadConfigurationXml(String configurationFile)
throws YarnException, IOException {
InputStream configurationInputStream =
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-8010. Add config in
FederationRMFailoverProxy to not bypass facade cache when failing over.
(Botong Huang via Subru).
Posted by na...@apache.org.
YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09999d7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09999d7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09999d7e
Branch: refs/heads/YARN-3409
Commit: 09999d7e014fde717e8b122773b68664f4594106
Parents: 725b10e
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Mar 28 11:33:19 2018 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Mar 28 11:33:19 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 9 ++-
.../yarn/conf/TestYarnConfigurationFields.java | 2 +
.../TestFederationRMFailoverProxyProvider.java | 81 +++++++++++++++++++-
.../FederationRMFailoverProxyProvider.java | 11 ++-
4 files changed, 94 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09999d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 04b2898..1f62bbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3089,15 +3089,18 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
FEDERATION_PREFIX + "cache-ttl.secs";
+ // 5 minutes
+ public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
+ public static final String FEDERATION_FLUSH_CACHE_FOR_RM_ADDR =
+ FEDERATION_PREFIX + "flush-cache-for-rm-addr";
+ public static final boolean DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR = true;
public static final String FEDERATION_REGISTRY_BASE_KEY =
FEDERATION_PREFIX + "registry.base-dir";
public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
"yarnfederation/";
- // 5 minutes
- public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
-
public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09999d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 9fe4f88..f4d1ac0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -79,6 +79,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
.add(YarnConfiguration.FEDERATION_FAILOVER_ENABLED);
configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS);
+ configurationPrefixToSkipCompare
+ .add(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR);
configurationPropsToSkipCompare
.add(YarnConfiguration.RM_EPOCH);
configurationPropsToSkipCompare
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09999d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
index e3f9155..0a7ee3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestFederationRMFailoverProxyProvider.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.federation.failover.FederationProxyProvider
import org.apache.hadoop.yarn.server.federation.failover.FederationRMFailoverProxyProvider;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
@@ -49,7 +50,11 @@ import org.junit.Before;
import org.junit.Test;
import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.atLeast;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
@@ -61,12 +66,20 @@ public class TestFederationRMFailoverProxyProvider {
private FederationStateStore stateStore;
private final String dummyCapability = "cap";
+ private GetClusterMetricsResponse threadResponse;
+
@Before
public void setUp() throws IOException, YarnException {
conf = new YarnConfiguration();
- stateStore = new MemoryFederationStateStore();
+
+ // Configure Facade cache to use a very long ttl
+ conf.setInt(YarnConfiguration.FEDERATION_CACHE_TIME_TO_LIVE_SECS, 60 * 60);
+
+ stateStore = spy(new MemoryFederationStateStore());
stateStore.init(conf);
FederationStateStoreFacade.getInstance().reinitialize(stateStore, conf);
+ verify(stateStore, times(0))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
}
@After
@@ -75,12 +88,25 @@ public class TestFederationRMFailoverProxyProvider {
stateStore = null;
}
- @Test
+ @Test(timeout = 60000)
public void testFederationRMFailoverProxyProvider() throws Exception {
+ testProxyProvider(true);
+ }
+
+ @Test (timeout=60000)
+ public void testFederationRMFailoverProxyProviderWithoutFlushFacadeCache()
+ throws Exception {
+ testProxyProvider(false);
+ }
+
+ private void testProxyProvider(boolean facadeFlushCache) throws Exception {
final SubClusterId subClusterId = SubClusterId.newInstance("SC-1");
final MiniYARNCluster cluster = new MiniYARNCluster(
"testFederationRMFailoverProxyProvider", 3, 0, 1, 1);
+ conf.setBoolean(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR,
+ facadeFlushCache);
+
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
conf.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
@@ -104,10 +130,16 @@ public class TestFederationRMFailoverProxyProvider {
.createRMProxy(conf, ApplicationClientProtocol.class, subClusterId,
UserGroupInformation.getCurrentUser());
+ verify(stateStore, times(1))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
+
// client will retry until the rm becomes active.
GetClusterMetricsResponse response =
client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+ verify(stateStore, times(1))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
+
// validate response
checkResponse(response);
@@ -118,7 +150,50 @@ public class TestFederationRMFailoverProxyProvider {
// Transition rm2 to active;
makeRMActive(subClusterId, cluster, 1);
- response = client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+
+ verify(stateStore, times(1))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
+
+ threadResponse = null;
+ Thread thread = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // In non flush cache case, we will be hitting the cache with old RM
+ // address and keep failing before the cache is flushed
+ threadResponse =
+ client.getClusterMetrics(GetClusterMetricsRequest.newInstance());
+ } catch (YarnException | IOException e) {
+ e.printStackTrace();
+ }
+ }
+ });
+ thread.start();
+
+ if (!facadeFlushCache) {
+ // Add a wait so that hopefully the thread has started hitting old cached
+ Thread.sleep(500);
+
+ // Should still be hitting cache
+ verify(stateStore, times(1))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
+
+ // Force flush cache, so that it will pick up the new RM address
+ FederationStateStoreFacade.getInstance().getSubCluster(subClusterId,
+ true);
+ }
+
+ // Wait for the thread to finish and grab result
+ thread.join();
+ response = threadResponse;
+
+ if (facadeFlushCache) {
+ verify(stateStore, atLeast(2))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
+ } else {
+ verify(stateStore, times(2))
+ .getSubClusters(any(GetSubClustersInfoRequest.class));
+ }
// validate response
checkResponse(response);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09999d7e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index c631208..cf6d1ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -64,7 +64,8 @@ public class FederationRMFailoverProxyProvider<T>
private FederationStateStoreFacade facade;
private SubClusterId subClusterId;
private UserGroupInformation originalUser;
- private boolean federationFailoverEnabled = false;
+ private boolean federationFailoverEnabled;
+ private boolean flushFacadeCacheForYarnRMAddr;
@Override
public void init(Configuration configuration, RMProxy<T> proxy,
@@ -75,13 +76,16 @@ public class FederationRMFailoverProxyProvider<T>
String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID);
Preconditions.checkNotNull(clusterId, "Missing RM ClusterId");
this.subClusterId = SubClusterId.newInstance(clusterId);
- this.facade = facade.getInstance();
+ this.facade = FederationStateStoreFacade.getInstance();
if (configuration instanceof YarnConfiguration) {
this.conf = (YarnConfiguration) configuration;
}
federationFailoverEnabled =
conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+ flushFacadeCacheForYarnRMAddr =
+ conf.getBoolean(YarnConfiguration.FEDERATION_FLUSH_CACHE_FOR_RM_ADDR,
+ YarnConfiguration.DEFAULT_FEDERATION_FLUSH_CACHE_FOR_RM_ADDR);
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
@@ -119,7 +123,8 @@ public class FederationRMFailoverProxyProvider<T>
try {
LOG.info("Failing over to the ResourceManager for SubClusterId: {}",
subClusterId);
- subClusterInfo = facade.getSubCluster(subClusterId, isFailover);
+ subClusterInfo = facade.getSubCluster(subClusterId,
+ this.flushFacadeCacheForYarnRMAddr && isFailover);
// updating the conf with the refreshed RM addresses as proxy
// creations are based out of conf
updateRMAddress(subClusterInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-7842. PB changes to carry
node-attributes in NM heartbeat. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-7842. PB changes to carry node-attributes in NM heartbeat. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c771007
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c771007
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c771007
Branch: refs/heads/YARN-3409
Commit: 9c771007360f23817757c5b585f77bbad9092903
Parents: 6308b86
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Jan 31 20:28:41 2018 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 08:17:15 2018 +0800
----------------------------------------------------------------------
.../protocolrecords/NodeHeartbeatRequest.java | 17 +++++++
.../impl/pb/NodeHeartbeatRequestPBImpl.java | 52 ++++++++++++++++++++
.../yarn_server_common_service_protos.proto | 5 ++
.../protocolrecords/TestProtocolRecords.java | 12 +++++
4 files changed, 86 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c771007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
index f238f79..4f99225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
public abstract class NodeHeartbeatRequest {
@@ -61,6 +62,18 @@ public abstract class NodeHeartbeatRequest {
return nodeHeartbeatRequest;
}
+ public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
+ MasterKey lastKnownContainerTokenMasterKey,
+ MasterKey lastKnownNMTokenMasterKey, Set<NodeLabel> nodeLabels,
+ Set<NodeAttribute> nodeAttributes,
+ Map<ApplicationId, AppCollectorData> registeringCollectors) {
+ NodeHeartbeatRequest request = NodeHeartbeatRequest
+ .newInstance(nodeStatus, lastKnownContainerTokenMasterKey,
+ lastKnownNMTokenMasterKey, nodeLabels, registeringCollectors);
+ request.setNodeAttributes(nodeAttributes);
+ return request;
+ }
+
public abstract NodeStatus getNodeStatus();
public abstract void setNodeStatus(NodeStatus status);
@@ -85,4 +98,8 @@ public abstract class NodeHeartbeatRequest {
public abstract void setRegisteringCollectors(Map<ApplicationId,
AppCollectorData> appCollectorsMap);
+
+ public abstract Set<NodeAttribute> getNodeAttributes();
+
+ public abstract void setNodeAttributes(Set<NodeAttribute> nodeAttributes);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c771007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index 1ffd223..c59127a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -27,6 +27,9 @@ import java.util.Map;
import java.util.Set;
import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
@@ -36,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorDataProto;
@@ -60,6 +64,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private MasterKey lastKnownContainerTokenMasterKey = null;
private MasterKey lastKnownNMTokenMasterKey = null;
private Set<NodeLabel> labels = null;
+ private Set<NodeAttribute> attributes = null;
private List<LogAggregationReport> logAggregationReportsForApps = null;
private Map<ApplicationId, AppCollectorData> registeringCollectors = null;
@@ -115,6 +120,15 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
}
builder.setNodeLabels(newBuilder.build());
}
+ if (this.attributes != null) {
+ builder.clearNodeAttributes();
+ YarnServerCommonServiceProtos.NodeAttributesProto.Builder attBuilder =
+ YarnServerCommonServiceProtos.NodeAttributesProto.newBuilder();
+ for (NodeAttribute attribute : attributes) {
+ attBuilder.addNodeAttributes(convertToProtoFormat(attribute));
+ }
+ builder.setNodeAttributes(attBuilder.build());
+ }
if (this.logAggregationReportsForApps != null) {
addLogAggregationStatusForAppsToProto();
}
@@ -372,6 +386,44 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
}
@Override
+ public Set<NodeAttribute> getNodeAttributes() {
+ initNodeAttributes();
+ return this.attributes;
+ }
+
+ private void initNodeAttributes() {
+ if (this.attributes != null) {
+ return;
+ }
+ NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNodeAttributes()) {
+ return;
+ }
+ YarnServerCommonServiceProtos.NodeAttributesProto nodeAttributes =
+ p.getNodeAttributes();
+ attributes = new HashSet<>();
+ for (NodeAttributeProto attributeProto :
+ nodeAttributes.getNodeAttributesList()) {
+ attributes.add(convertFromProtoFormat(attributeProto));
+ }
+ }
+
+ @Override
+ public void setNodeAttributes(Set<NodeAttribute> nodeAttributes) {
+ maybeInitBuilder();
+ builder.clearNodeAttributes();
+ this.attributes = nodeAttributes;
+ }
+
+ private NodeAttributePBImpl convertFromProtoFormat(NodeAttributeProto p) {
+ return new NodeAttributePBImpl(p);
+ }
+
+ private NodeAttributeProto convertToProtoFormat(NodeAttribute attribute) {
+ return ((NodeAttributePBImpl) attribute).getProto();
+ }
+
+ @Override
public List<LogAggregationReport> getLogAggregationReportsForApps() {
if (this.logAggregationReportsForApps != null) {
return this.logAggregationReportsForApps;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c771007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 1b090bf..4e7d850 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -57,6 +57,10 @@ message NodeLabelsProto {
repeated NodeLabelProto nodeLabels = 1;
}
+message NodeAttributesProto {
+ repeated NodeAttributeProto nodeAttributes = 1;
+}
+
message RegisterNodeManagerRequestProto {
optional NodeIdProto node_id = 1;
optional int32 http_port = 3;
@@ -94,6 +98,7 @@ message NodeHeartbeatRequestProto {
optional NodeLabelsProto nodeLabels = 4;
repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5;
repeated AppCollectorDataProto registering_collectors = 6;
+ optional NodeAttributesProto nodeAttributes = 7;
}
message LogAggregationReportProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c771007/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
index 74f19e5..e6e79d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/api/protocolrecords/TestProtocolRecords.java
@@ -24,7 +24,9 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
+import com.google.common.collect.Sets;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -39,6 +41,8 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
@@ -173,6 +177,13 @@ public class TestProtocolRecords {
nodeStatus.setOpportunisticContainersStatus(opportunisticContainersStatus);
record.setNodeStatus(nodeStatus);
+ Set<NodeAttribute> attributeSet =
+ Sets.newHashSet(NodeAttribute.newInstance("attributeA",
+ NodeAttributeType.STRING, "valueA"),
+ NodeAttribute.newInstance("attributeB",
+ NodeAttributeType.STRING, "valueB"));
+ record.setNodeAttributes(attributeSet);
+
NodeHeartbeatRequestPBImpl pb = new
NodeHeartbeatRequestPBImpl(
((NodeHeartbeatRequestPBImpl) record).getProto());
@@ -183,6 +194,7 @@ public class TestProtocolRecords {
Assert.assertEquals(321,
pb.getNodeStatus().getOpportunisticContainersStatus()
.getWaitQueueLength());
+ Assert.assertEquals(2, pb.getNodeAttributes().size());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: YARN-8094. Support configuration
based Node Attribute provider. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-8094. Support configuration based Node Attribute provider. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33b67d03
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33b67d03
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33b67d03
Branch: refs/heads/YARN-3409
Commit: 33b67d03272e69c9e901a1eae75b2fb0a4a4eed1
Parents: 3a24790
Author: Sunil G <su...@apache.org>
Authored: Sat Mar 31 19:53:06 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 11:43:16 2018 +0800
----------------------------------------------------------------------
.../ConfigurationNodeAttributesProvider.java | 70 +++++++++++++++++-
.../ScriptBasedNodeAttributesProvider.java | 8 +++
...TestConfigurationNodeAttributesProvider.java | 74 ++++++++++++++++++++
3 files changed, 150 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33b67d03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
index 74341eb..ab8a8b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
@@ -18,13 +18,19 @@
package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableSet;
+import org.apache.commons.lang3.EnumUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.TimerTask;
import java.util.Set;
@@ -38,6 +44,9 @@ public class ConfigurationNodeAttributesProvider
private static final Logger LOG =
LoggerFactory.getLogger(ConfigurationNodeAttributesProvider.class);
+ private static final String NODE_ATTRIBUTES_DELIMITER = ":";
+ private static final String NODE_ATTRIBUTE_DELIMITER = ",";
+
public ConfigurationNodeAttributesProvider() {
super("Configuration Based Node Attributes Provider");
}
@@ -59,11 +68,68 @@ public class ConfigurationNodeAttributesProvider
setDescriptors(parseAttributes(configuredNodeAttributes));
}
- // TODO parse attributes from configuration
@VisibleForTesting
public Set<NodeAttribute> parseAttributes(String config)
throws IOException {
- return new HashSet<>();
+ if (Strings.isNullOrEmpty(config)) {
+ return ImmutableSet.of();
+ }
+ Set<NodeAttribute> attributeSet = new HashSet<>();
+ // Configuration value should be in one line, format:
+ // "ATTRIBUTE_NAME,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE",
+ // multiple node-attributes are delimited by ":".
+ // Each attribute str should not container any space.
+ String[] attributeStrs = config.split(NODE_ATTRIBUTES_DELIMITER);
+ for (String attributeStr : attributeStrs) {
+ String[] fields = attributeStr.split(NODE_ATTRIBUTE_DELIMITER);
+ if (fields.length != 3) {
+ throw new IOException("Invalid value for "
+ + YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
+ + "=" + config);
+ }
+
+ // We don't allow user config to overwrite our dist prefix,
+ // so disallow any prefix set in the configuration.
+ if (fields[0].contains("/")) {
+ throw new IOException("Node attribute set in "
+ + YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
+ + " should not contain any prefix.");
+ }
+
+ // Make sure attribute type is valid.
+ if (!EnumUtils.isValidEnum(NodeAttributeType.class, fields[1])) {
+ throw new IOException("Invalid node attribute type: "
+ + fields[1] + ", valid values are "
+ + Arrays.asList(NodeAttributeType.values()));
+ }
+
+ // Automatically setup prefix for collected attributes
+ NodeAttribute na = NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_DISTRIBUTED,
+ fields[0],
+ NodeAttributeType.valueOf(fields[1]),
+ fields[2]);
+
+ // Since a NodeAttribute is identical with another one as long as
+ // their prefix and name are same, to avoid attributes getting
+ // overwritten by ambiguous attribute, make sure it fails in such
+ // case.
+ if (!attributeSet.add(na)) {
+ throw new IOException("Ambiguous node attribute is found: "
+ + na.toString() + ", a same attribute already exists");
+ }
+ }
+
+ // Before updating the attributes to the provider,
+ // verify if they are valid
+ try {
+ NodeLabelUtil.validateNodeAttributes(attributeSet);
+ } catch (IOException e) {
+ throw new IOException("Node attributes set by configuration property: "
+ + YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES
+ + " is not valid. Detail message: " + e.getMessage());
+ }
+ return attributeSet;
}
private class ConfigurationMonitorTimerTask extends TimerTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33b67d03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
index 4621434..7e5aefc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
@@ -117,6 +117,14 @@ public class ScriptBasedNodeAttributesProvider extends NodeAttributesProvider{
+ NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_VALUE; but get "
+ nodeAttribute);
}
+
+ // We don't allow script to overwrite our dist prefix,
+ // so disallow any prefix set in the script.
+ if (attributeStrs[0].contains("/")) {
+ throw new IOException("Node attributes reported by script"
+ + " should not contain any prefix.");
+ }
+
// Automatically setup prefix for collected attributes
NodeAttribute na = NodeAttribute
.newInstance(NodeAttribute.PREFIX_DISTRIBUTED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33b67d03/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
index 54cc8f0..d4384b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
@@ -36,6 +36,7 @@ import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Set;
import java.util.ArrayList;
import java.util.concurrent.TimeoutException;
@@ -182,4 +183,77 @@ public class TestConfigurationNodeAttributesProvider {
Assert.fail("Expecting a failure in previous check!");
}
+
+ @Test
+ public void testFetchAttributesFromConfiguration() {
+ Configuration conf = new Configuration();
+ // Set fetch interval to -1 to disable refresh.
+ conf.setLong(
+ YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS, -1);
+ conf.setStrings(
+ YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES, "");
+ }
+
+ @Test
+ public void testParseConfiguration() throws IOException {
+ // ATTRIBUTE_NAME,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE
+ String attributesStr = "hostname,STRING,host1234:uptime,STRING,321543";
+ Set<NodeAttribute> attributes = nodeAttributesProvider
+ .parseAttributes(attributesStr);
+ Assert.assertEquals(2, attributes.size());
+ Iterator<NodeAttribute> ait = attributes.iterator();
+
+ while(ait.hasNext()) {
+ NodeAttribute at = ait.next();
+ if (at.getAttributeName().equals("hostname")) {
+ Assert.assertEquals("hostname", at.getAttributeName());
+ Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED,
+ at.getAttributePrefix());
+ Assert.assertEquals(NodeAttributeType.STRING,
+ at.getAttributeType());
+ Assert.assertEquals("host1234", at.getAttributeValue());
+ } else if (at.getAttributeName().equals("uptime")) {
+ Assert.assertEquals("uptime", at.getAttributeName());
+ Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED,
+ at.getAttributePrefix());
+ Assert.assertEquals(NodeAttributeType.STRING,
+ at.getAttributeType());
+ Assert.assertEquals("321543", at.getAttributeValue());
+ } else {
+ Assert.fail("Unexpected attribute");
+ }
+ }
+ // Missing type
+ attributesStr = "hostname,host1234";
+ try {
+ nodeAttributesProvider.parseAttributes(attributesStr);
+ Assert.fail("Expecting a parsing failure");
+ } catch (IOException e) {
+ Assert.assertNotNull(e);
+ Assert.assertTrue(e.getMessage().contains("Invalid value"));
+ }
+
+ // Extra prefix
+ attributesStr = "prefix/hostname,STRING,host1234";
+ try {
+ nodeAttributesProvider.parseAttributes(attributesStr);
+ Assert.fail("Expecting a parsing failure");
+ } catch (IOException e) {
+ Assert.assertNotNull(e);
+ Assert.assertTrue(e.getMessage()
+ .contains("should not contain any prefix."));
+ }
+
+ // Invalid type
+ attributesStr = "hostname,T,host1234";
+ try {
+ nodeAttributesProvider.parseAttributes(attributesStr);
+ Assert.fail("Expecting a parsing failure");
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert.assertNotNull(e);
+ Assert.assertTrue(e.getMessage()
+ .contains("Invalid node attribute type"));
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-7965. NodeAttributeManager
add/get API is not working properly. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-7965. NodeAttributeManager add/get API is not working properly. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cca30fd3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cca30fd3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cca30fd3
Branch: refs/heads/YARN-3409
Commit: cca30fd37fe08f0d65c69ca59dccdaf4c90e377a
Parents: 14a9250
Author: Naganarasimha <na...@apache.org>
Authored: Tue Feb 27 18:46:16 2018 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 11:30:55 2018 +0800
----------------------------------------------------------------------
.../yarn/nodelabels/NodeAttributesManager.java | 8 +-
.../nodelabels/NodeAttributesManagerImpl.java | 38 ++-
.../nodelabels/TestNodeAttributesManager.java | 258 +++++++++++++++++++
3 files changed, 288 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cca30fd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
index 63f3dcf..effda9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -69,10 +69,14 @@ public abstract class NodeAttributesManager extends AbstractService {
Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
/**
+ * Returns a set of node attributes whose prefix is one of the given
+ * prefixes; if the prefix set is null or empty, all attributes are returned;
+ * if prefix set is given but no mapping could be found, an empty set
+ * is returned.
+ *
* @param prefix set of prefix string's for which the attributes needs to
* returned
- * @return set of node Attributes objects for the specified set of prefixes,
- * else return all
+ * @return set of node Attributes
*/
public abstract Set<NodeAttribute> getClusterNodeAttributes(
Set<String> prefix);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cca30fd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index 2e63a7c..a902ac6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -27,6 +27,7 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentHashMap.KeySetView;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
@@ -141,6 +142,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
Host node = nodeCollections.get(nodeHost);
if (node == null) {
node = new Host(nodeHost);
+ nodeCollections.put(nodeHost, node);
}
switch (op) {
case REMOVE:
@@ -181,8 +183,16 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void removeNodeFromAttributes(String nodeHost,
Set<NodeAttribute> attributeMappings) {
- for (NodeAttribute attribute : attributeMappings) {
- clusterAttributes.get(attribute).removeNode(nodeHost);
+ for (NodeAttribute rmAttribute : attributeMappings) {
+ RMNodeAttribute host = clusterAttributes.get(rmAttribute);
+ if (host != null) {
+ host.removeNode(nodeHost);
+ // If there is no other host has such attribute,
+ // remove it from the global mapping.
+ if (host.getAssociatedNodeIds().isEmpty()) {
+ clusterAttributes.remove(rmAttribute);
+ }
+ }
}
}
@@ -305,19 +315,19 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
@Override
public Set<NodeAttribute> getClusterNodeAttributes(Set<String> prefix) {
Set<NodeAttribute> attributes = new HashSet<>();
- try {
- readLock.lock();
- attributes.addAll(clusterAttributes.keySet());
- } finally {
- readLock.unlock();
+ KeySetView<NodeAttribute, RMNodeAttribute> allAttributes =
+ clusterAttributes.keySet();
+ // Return all if prefix is not given.
+ if (prefix == null || prefix.isEmpty()) {
+ attributes.addAll(allAttributes);
+ return attributes;
}
- if (prefix != null && prefix.isEmpty()) {
- Iterator<NodeAttribute> iterator = attributes.iterator();
- while (iterator.hasNext()) {
- NodeAttribute attribute = iterator.next();
- if (!prefix.contains(attribute.getAttributePrefix())) {
- iterator.remove();
- }
+ // Try search attributes by prefix and return valid ones.
+ Iterator<NodeAttribute> iterator = allAttributes.iterator();
+ while (iterator.hasNext()) {
+ NodeAttribute current = iterator.next();
+ if (prefix.contains(current.getAttributePrefix())) {
+ attributes.add(current);
}
}
return attributes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cca30fd3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
new file mode 100644
index 0000000..b639a74
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
@@ -0,0 +1,258 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.nodelabels.AttributeValue;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
+import org.junit.Test;
+import org.junit.Before;
+import org.junit.After;
+import org.junit.Assert;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Unit tests for node attribute manager.
+ */
+public class TestNodeAttributesManager {
+
+ private NodeAttributesManager attributesManager;
+ private final static String[] PREFIXES =
+ new String[] {"yarn.test1.io", "yarn.test2.io", "yarn.test3.io"};
+ private final static String[] HOSTNAMES =
+ new String[] {"host1", "host2", "host3"};
+
+ @Before
+ public void init() {
+ Configuration conf = new Configuration();
+ attributesManager = new NodeAttributesManagerImpl();
+ attributesManager.init(conf);
+ attributesManager.start();
+ }
+
+ @After
+ public void cleanUp() {
+ if (attributesManager != null) {
+ attributesManager.stop();
+ }
+ }
+
+ private Set<NodeAttribute> createAttributesForTest(String attributePrefix,
+ int numOfAttributes, String attributeNamePrefix,
+ String attributeValuePrefix) {
+ Set<NodeAttribute> attributes = new HashSet<>();
+ for (int i = 0; i< numOfAttributes; i++) {
+ NodeAttribute attribute = NodeAttribute.newInstance(
+ attributePrefix, attributeNamePrefix + "_" + i,
+ NodeAttributeType.STRING, attributeValuePrefix + "_" + i);
+ attributes.add(attribute);
+ }
+ return attributes;
+ }
+
+ private boolean sameAttributeSet(Set<NodeAttribute> set1,
+ Set<NodeAttribute> set2) {
+ return Sets.difference(set1, set2).isEmpty();
+ }
+
+ @Test
+ public void testAddNodeAttributes() throws IOException {
+ Map<String, Set<NodeAttribute>> toAddAttributes = new HashMap<>();
+ Map<NodeAttribute, AttributeValue> nodeAttributes;
+
+ // Add 3 attributes to host1
+ // yarn.test1.io/A1=host1_v1_1
+ // yarn.test1.io/A2=host1_v1_2
+ // yarn.test1.io/A3=host1_v1_3
+ toAddAttributes.put(HOSTNAMES[0],
+ createAttributesForTest(PREFIXES[0], 3, "A", "host1_v1"));
+
+ attributesManager.addNodeAttributes(toAddAttributes);
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+
+ Assert.assertEquals(3, nodeAttributes.size());
+ Assert.assertTrue(sameAttributeSet(toAddAttributes.get(HOSTNAMES[0]),
+ nodeAttributes.keySet()));
+
+ // Add 2 attributes to host2
+ // yarn.test1.io/A1=host2_v1_1
+ // yarn.test1.io/A2=host2_v1_2
+ toAddAttributes.clear();
+ toAddAttributes.put(HOSTNAMES[1],
+ createAttributesForTest(PREFIXES[0], 2, "A", "host2_v1"));
+ attributesManager.addNodeAttributes(toAddAttributes);
+
+ // Verify host1 attributes are still valid.
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(3, nodeAttributes.size());
+
+ // Verify new added host2 attributes are correctly updated.
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[1]);
+ Assert.assertEquals(2, nodeAttributes.size());
+ Assert.assertTrue(sameAttributeSet(toAddAttributes.get(HOSTNAMES[1]),
+ nodeAttributes.keySet()));
+
+ // Cluster wide, it only has 3 attributes.
+ // yarn.test1.io/A1
+ // yarn.test1.io/A2
+ // yarn.test1.io/A3
+ Set<NodeAttribute> clusterAttributes = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[0]));
+ Assert.assertEquals(3, clusterAttributes.size());
+
+ // Query for attributes under a non-exist prefix,
+ // ensure it returns an empty set.
+ clusterAttributes = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet("non_exist_prefix"));
+ Assert.assertEquals(0, clusterAttributes.size());
+
+ // Not provide any prefix, ensure it returns all attributes.
+ clusterAttributes = attributesManager.getClusterNodeAttributes(null);
+ Assert.assertEquals(3, clusterAttributes.size());
+
+ // Add some other attributes with different prefixes on host1 and host2.
+ toAddAttributes.clear();
+
+ // Host1
+ // yarn.test2.io/A_1=host1_v2_1
+ // ...
+ // yarn.test2.io/A_10=host1_v2_10
+ toAddAttributes.put(HOSTNAMES[0],
+ createAttributesForTest(PREFIXES[1], 10, "C", "host1_v2"));
+ // Host2
+ // yarn.test2.io/C_1=host1_v2_1
+ // ...
+ // yarn.test2.io/C_20=host1_v2_20
+ toAddAttributes.put(HOSTNAMES[1],
+ createAttributesForTest(PREFIXES[1], 20, "C", "host1_v2"));
+ attributesManager.addNodeAttributes(toAddAttributes);
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(13, nodeAttributes.size());
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[1]);
+ Assert.assertEquals(22, nodeAttributes.size());
+ }
+
+ @Test
+ public void testRemoveNodeAttributes() throws IOException {
+ Map<String, Set<NodeAttribute>> toAddAttributes = new HashMap<>();
+ Map<String, Set<NodeAttribute>> toRemoveAttributes = new HashMap<>();
+ Set<NodeAttribute> allAttributesPerPrefix = new HashSet<>();
+ Map<NodeAttribute, AttributeValue> nodeAttributes;
+
+ // Host1 -----------------------
+ // yarn.test1.io
+ // A1=host1_v1_1
+ // A2=host1_v1_2
+ // A3=host1_v1_3
+ // yarn.test2.io
+ // B1=host1_v2_1
+ // ...
+ // B5=host5_v2_5
+ // Host2 -----------------------
+ // yarn.test1.io
+ // A1=host2_v1_1
+ // A2=host2_v1_2
+ // yarn.test3.io
+ // C1=host2_v3_1
+ // c2=host2_v3_2
+ Set<NodeAttribute> host1set = new HashSet<>();
+ Set<NodeAttribute> host1set1 =
+ createAttributesForTest(PREFIXES[0], 3, "A", "host1_v1");
+ Set<NodeAttribute> host1set2 =
+ createAttributesForTest(PREFIXES[1], 5, "B", "host1_v1");
+ host1set.addAll(host1set1);
+ host1set.addAll(host1set2);
+
+ Set<NodeAttribute> host2set = new HashSet<>();
+ Set<NodeAttribute> host2set1 =
+ createAttributesForTest(PREFIXES[0], 2, "A", "host2_v1");
+ Set<NodeAttribute> host2set2 =
+ createAttributesForTest(PREFIXES[2], 2, "C", "host2_v3");
+ host2set.addAll(host2set1);
+ host2set.addAll(host2set2);
+
+ toAddAttributes.put(HOSTNAMES[0], host1set);
+ toAddAttributes.put(HOSTNAMES[1], host2set);
+ attributesManager.addNodeAttributes(toAddAttributes);
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(8, nodeAttributes.size());
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[1]);
+ Assert.assertEquals(4, nodeAttributes.size());
+
+ allAttributesPerPrefix = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[0]));
+ Assert.assertEquals(3, allAttributesPerPrefix.size());
+ allAttributesPerPrefix = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[1]));
+ Assert.assertEquals(5, allAttributesPerPrefix.size());
+ allAttributesPerPrefix = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[2]));
+ Assert.assertEquals(2, allAttributesPerPrefix.size());
+
+ // Remove "yarn.test1.io/A_2" from host1
+ Set<NodeAttribute> attributes2rm1 = new HashSet<>();
+ attributes2rm1.add(NodeAttribute.newInstance(PREFIXES[0], "A_2",
+ NodeAttributeType.STRING, "anyValue"));
+ toRemoveAttributes.put(HOSTNAMES[0], attributes2rm1);
+ attributesManager.removeNodeAttributes(toRemoveAttributes);
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(7, nodeAttributes.size());
+
+ // Remove again, but give a non-exist attribute name
+ attributes2rm1.clear();
+ toRemoveAttributes.clear();
+ attributes2rm1.add(NodeAttribute.newInstance(PREFIXES[0], "non_exist_name",
+ NodeAttributeType.STRING, "anyValue"));
+ toRemoveAttributes.put(HOSTNAMES[0], attributes2rm1);
+ attributesManager.removeNodeAttributes(toRemoveAttributes);
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(7, nodeAttributes.size());
+
+ // Remove "yarn.test1.io/A_2" from host2 too,
+ // by then there will be no such attribute exist in the cluster.
+ Set<NodeAttribute> attributes2rm2 = new HashSet<>();
+ attributes2rm2.add(NodeAttribute.newInstance(PREFIXES[0], "A_2",
+ NodeAttributeType.STRING, "anyValue"));
+ toRemoveAttributes.clear();
+ toRemoveAttributes.put(HOSTNAMES[1], attributes2rm2);
+ attributesManager.removeNodeAttributes(toRemoveAttributes);
+
+ // Make sure cluster wide attributes are still consistent.
+ // Since both host1 and host2 doesn't have "yarn.test1.io/A_2",
+ // get all attributes under prefix "yarn.test1.io" should only return
+ // us A_1 and A_3.
+ allAttributesPerPrefix = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[0]));
+ Assert.assertEquals(2, allAttributesPerPrefix.size());
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-8085. ResourceProfilesManager
should be set in RMActiveServiceContext. Contributed by Tao Yang.
Posted by na...@apache.org.
YARN-8085. ResourceProfilesManager should be set in RMActiveServiceContext. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a59d60e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a59d60e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a59d60e
Branch: refs/heads/YARN-3409
Commit: 7a59d60e0c50e8264e07f6b1dadaeedba676c8c2
Parents: ad10cbd
Author: Sunil G <su...@apache.org>
Authored: Thu Mar 29 21:41:16 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Mar 29 21:41:16 2018 +0530
----------------------------------------------------------------------
.../resourcemanager/RMActiveServiceContext.java | 11 ++++++
.../server/resourcemanager/RMContextImpl.java | 6 +--
.../server/resourcemanager/ResourceManager.java | 13 +++----
.../yarn/server/resourcemanager/TestRMHA.java | 39 ++++++++++++++++++++
4 files changed, 58 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a59d60e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 06a1d00..66065e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystem;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceProfilesManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.monitor.RMAppLifetimeMonitor;
@@ -111,6 +112,7 @@ public class RMActiveServiceContext {
private QueueLimitCalculator queueLimitCalculator;
private AllocationTagsManager allocationTagsManager;
private PlacementConstraintManager placementConstraintManager;
+ private ResourceProfilesManager resourceProfilesManager;
public RMActiveServiceContext() {
queuePlacementManager = new PlacementManager();
@@ -513,4 +515,13 @@ public class RMActiveServiceContext {
QueueLimitCalculator limitCalculator) {
this.queueLimitCalculator = limitCalculator;
}
+
+ public ResourceProfilesManager getResourceProfilesManager() {
+ return resourceProfilesManager;
+ }
+
+ public void setResourceProfilesManager(
+ ResourceProfilesManager resourceProfilesManager) {
+ this.resourceProfilesManager = resourceProfilesManager;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a59d60e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 0b6be72..84e0f6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -93,8 +93,6 @@ public class RMContextImpl implements RMContext {
*/
private RMActiveServiceContext activeServiceContext;
- private ResourceProfilesManager resourceProfilesManager;
-
private String proxyHostAndPort = null;
/**
@@ -591,7 +589,7 @@ public class RMContextImpl implements RMContext {
@Override
public ResourceProfilesManager getResourceProfilesManager() {
- return this.resourceProfilesManager;
+ return this.activeServiceContext.getResourceProfilesManager();
}
String getProxyHostAndPort(Configuration conf) {
@@ -619,7 +617,7 @@ public class RMContextImpl implements RMContext {
@Override
public void setResourceProfilesManager(ResourceProfilesManager mgr) {
- this.resourceProfilesManager = mgr;
+ this.activeServiceContext.setResourceProfilesManager(mgr);
}
// Note: Read java doc before adding any services over here.
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a59d60e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 38da7f5..733da5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -245,13 +245,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
this.rmContext = new RMContextImpl();
rmContext.setResourceManager(this);
-
- // add resource profiles here because it's used by AbstractYarnScheduler
- ResourceProfilesManager resourceProfilesManager =
- createResourceProfileManager();
- resourceProfilesManager.init(conf);
- rmContext.setResourceProfilesManager(resourceProfilesManager);
-
this.configurationProvider =
ConfigurationProviderFactory.getConfigurationProvider(conf);
this.configurationProvider.init(this.conf);
@@ -655,6 +648,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
addService(placementConstraintManager);
rmContext.setPlacementConstraintManager(placementConstraintManager);
+ // add resource profiles here because it's used by AbstractYarnScheduler
+ ResourceProfilesManager resourceProfilesManager =
+ createResourceProfileManager();
+ resourceProfilesManager.init(conf);
+ rmContext.setResourceProfilesManager(resourceProfilesManager);
+
RMDelegatedNodeLabelsUpdater delegatedNodeLabelsUpdater =
createRMDelegatedNodeLabelsUpdater();
if (delegatedNodeLabelsUpdater != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a59d60e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
index 20e9ff4..385e8db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHA.java
@@ -658,6 +658,45 @@ public class TestRMHA {
assertEquals(HAServiceState.STANDBY, rm.getRMContext().getHAServiceState());
}
+ @Test
+ public void testResourceProfilesManagerAfterRMWentStandbyThenBackToActive()
+ throws Exception {
+ configuration.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
+ configuration.setBoolean(YarnConfiguration.RECOVERY_ENABLED, true);
+ Configuration conf = new YarnConfiguration(configuration);
+ conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+
+ // 1. start RM
+ rm = new MockRM(conf);
+ rm.init(conf);
+ rm.start();
+
+ StateChangeRequestInfo requestInfo = new StateChangeRequestInfo(
+ HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ checkMonitorHealth();
+ checkStandbyRMFunctionality();
+
+ // 2. Transition to active
+ rm.adminService.transitionToActive(requestInfo);
+ checkMonitorHealth();
+ checkActiveRMFunctionality();
+
+ // 3. Transition to standby
+ rm.adminService.transitionToStandby(requestInfo);
+ checkMonitorHealth();
+ checkStandbyRMFunctionality();
+
+ // 4. Transition to active
+ rm.adminService.transitionToActive(requestInfo);
+ checkMonitorHealth();
+ checkActiveRMFunctionality();
+
+ // 5. Check ResourceProfilesManager
+ Assert.assertNotNull(
+ "ResourceProfilesManager should not be null!",
+ rm.getRMContext().getResourceProfilesManager());
+ }
+
public void innerTestHAWithRMHostName(boolean includeBindHost) {
//this is run two times, with and without a bind host configured
if (includeBindHost) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-7734. Fix UT failure
TestContainerLogsPage#testContainerLogPageAccess. Contributed by Tao Yang.
Posted by na...@apache.org.
YARN-7734. Fix UT failure TestContainerLogsPage#testContainerLogPageAccess. Contributed by Tao Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/411993f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/411993f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/411993f6
Branch: refs/heads/YARN-3409
Commit: 411993f6e5723c8cba8100bff0269418e46f6367
Parents: a71656c
Author: Weiwei Yang <ww...@apache.org>
Authored: Wed Mar 28 17:35:17 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Wed Mar 28 17:35:38 2018 +0800
----------------------------------------------------------------------
.../yarn/server/nodemanager/webapp/TestContainerLogsPage.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/411993f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 8dc06c7..ece1af4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
@@ -259,6 +259,7 @@ public class TestContainerLogsPage {
new ConcurrentHashMap<ContainerId, Container>();
when(context.getContainers()).thenReturn(containers);
when(context.getLocalDirsHandler()).thenReturn(dirsHandler);
+ when(context.getConf()).thenReturn(conf);
MockContainer container = new MockContainer(appAttemptId,
new AsyncDispatcher(), conf, user, appId, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: HDFS-13289. RBF:
TestConnectionManager#testCleanup() test case need correction. Contributed by
Dibyendu Karmakar.
Posted by na...@apache.org.
HDFS-13289. RBF: TestConnectionManager#testCleanup() test case need correction. Contributed by Dibyendu Karmakar.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc8e3432
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc8e3432
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc8e3432
Branch: refs/heads/YARN-3409
Commit: dc8e3432013153ac11d31d6b462aa96f8ca2c443
Parents: acfd764
Author: Inigo Goiri <in...@apache.org>
Authored: Sat Mar 31 09:46:13 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Sat Mar 31 09:46:13 2018 -0700
----------------------------------------------------------------------
.../router/TestConnectionManager.java | 22 +++++++++++++++-----
1 file changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc8e3432/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
index 741d1f6..2e4b80d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestConnectionManager.java
@@ -29,6 +29,7 @@ import java.util.Map;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/**
* Test functionalities of {@link ConnectionManager}, which manages a pool
@@ -94,14 +95,20 @@ public class TestConnectionManager {
// Make sure the number of connections doesn't go below minSize
ConnectionPool pool3 = new ConnectionPool(
conf, TEST_NN_ADDRESS, TEST_USER3, 2, 10);
- addConnectionsToPool(pool3, 10, 0);
- poolMap.put(new ConnectionPoolId(TEST_USER2, TEST_NN_ADDRESS), pool3);
- connManager.cleanup(pool3);
+ addConnectionsToPool(pool3, 8, 0);
+ poolMap.put(new ConnectionPoolId(TEST_USER3, TEST_NN_ADDRESS), pool3);
+ checkPoolConnections(TEST_USER3, 10, 0);
+ for (int i = 0; i < 10; i++) {
+ connManager.cleanup(pool3);
+ }
checkPoolConnections(TEST_USER3, 2, 0);
// With active connections added to pool, make sure it honors the
// MIN_ACTIVE_RATIO again
- addConnectionsToPool(pool3, 10, 2);
- connManager.cleanup(pool3);
+ addConnectionsToPool(pool3, 8, 2);
+ checkPoolConnections(TEST_USER3, 10, 2);
+ for (int i = 0; i < 10; i++) {
+ connManager.cleanup(pool3);
+ }
checkPoolConnections(TEST_USER3, 4, 2);
}
@@ -145,13 +152,18 @@ public class TestConnectionManager {
private void checkPoolConnections(UserGroupInformation ugi,
int numOfConns, int numOfActiveConns) {
+ boolean connPoolFoundForUser = false;
for (Map.Entry<ConnectionPoolId, ConnectionPool> e :
connManager.getPools().entrySet()) {
if (e.getKey().getUgi() == ugi) {
assertEquals(numOfConns, e.getValue().getNumConnections());
assertEquals(numOfActiveConns, e.getValue().getNumActiveConnections());
+ connPoolFoundForUser = true;
}
}
+ if (!connPoolFoundForUser) {
+ fail("Connection pool not found for user " + ugi.getUserName());
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-8010. Add config in
FederationRMFailoverProxy to not bypass facade cache when failing over.
(Botong Huang via Subru).
Posted by na...@apache.org.
YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2a2ef15c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2a2ef15c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2a2ef15c
Branch: refs/heads/YARN-3409
Commit: 2a2ef15caf791f30c471526c1b74e68803f0c405
Parents: 3fe41c6
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Mar 27 17:39:46 2018 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Mar 27 17:39:46 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 9 ++++++---
.../src/main/resources/yarn-default.xml | 9 +++++++++
.../failover/FederationRMFailoverProxyProvider.java | 11 ++++++++---
3 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a2ef15c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 04b2898..658099a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3089,15 +3089,18 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
FEDERATION_PREFIX + "cache-ttl.secs";
+ // 5 minutes
+ public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
+ public static final String FEDERATION_FLUSh_CACHE_FOR_RM_ADDR =
+ FEDERATION_PREFIX + "flush-cache-for-rm-addr";
+ public static final boolean DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = true;
public static final String FEDERATION_REGISTRY_BASE_KEY =
FEDERATION_PREFIX + "registry.base-dir";
public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
"yarnfederation/";
- // 5 minutes
- public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
-
public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a2ef15c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 114ba4b..2eba8df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2925,6 +2925,15 @@
</property>
<property>
+ <description>
+ Whether to flush FederationStateStoreFacade cache to get subcluster info
+ when FederationRMFailoverProxyProvider is performing failover.
+ </description>
+ <name>yarn.federation.flush-cache-for-rm-addr</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>The registry base directory for federation.</description>
<name>yarn.federation.registry.base-dir</name>
<value>yarnfederation/</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2a2ef15c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index c631208..b72b199 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -64,7 +64,8 @@ public class FederationRMFailoverProxyProvider<T>
private FederationStateStoreFacade facade;
private SubClusterId subClusterId;
private UserGroupInformation originalUser;
- private boolean federationFailoverEnabled = false;
+ private boolean federationFailoverEnabled;
+ private boolean flushFacadeCacheForYarnRMAddr;
@Override
public void init(Configuration configuration, RMProxy<T> proxy,
@@ -75,13 +76,16 @@ public class FederationRMFailoverProxyProvider<T>
String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID);
Preconditions.checkNotNull(clusterId, "Missing RM ClusterId");
this.subClusterId = SubClusterId.newInstance(clusterId);
- this.facade = facade.getInstance();
+ this.facade = FederationStateStoreFacade.getInstance();
if (configuration instanceof YarnConfiguration) {
this.conf = (YarnConfiguration) configuration;
}
federationFailoverEnabled =
conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
+ flushFacadeCacheForYarnRMAddr =
+ conf.getBoolean(YarnConfiguration.FEDERATION_FLUSh_CACHE_FOR_RM_ADDR,
+ YarnConfiguration.DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR);
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
@@ -119,7 +123,8 @@ public class FederationRMFailoverProxyProvider<T>
try {
LOG.info("Failing over to the ResourceManager for SubClusterId: {}",
subClusterId);
- subClusterInfo = facade.getSubCluster(subClusterId, isFailover);
+ subClusterInfo = facade.getSubCluster(subClusterId,
+ this.flushFacadeCacheForYarnRMAddr && isFailover);
// updating the conf with the refreshed RM addresses as proxy
// creations are based out of conf
updateRMAddress(subClusterInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-7871. Node attributes
reporting from NM to RM. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-7871. Node attributes reporting from NM to RM. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5bd044e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5bd044e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5bd044e6
Branch: refs/heads/YARN-3409
Commit: 5bd044e6923367be8f5bbade22850478abc4dc27
Parents: cca30fd
Author: Naganarasimha <na...@apache.org>
Authored: Mon Mar 12 08:05:53 2018 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 11:43:16 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +-
.../yarn/nodelabels/NodeAttributesManager.java | 17 +-
.../hadoop/yarn/nodelabels/NodeLabelUtil.java | 19 ++
.../src/main/resources/yarn-default.xml | 24 +++
.../yarn/server/nodemanager/NodeManager.java | 70 +++++--
.../server/nodemanager/NodeStatusUpdater.java | 14 ++
.../nodemanager/NodeStatusUpdaterImpl.java | 70 ++++++-
.../ConfigurationNodeAttributesProvider.java | 90 +++++++++
.../server/nodemanager/TestNodeManager.java | 2 +-
.../TestNodeStatusUpdaterForLabels.java | 10 +-
...TestConfigurationNodeAttributesProvider.java | 185 +++++++++++++++++++
.../resourcemanager/ResourceTrackerService.java | 30 +++
.../nodelabels/NodeAttributesManagerImpl.java | 52 ++++--
.../TestResourceTrackerService.java | 78 ++++++++
.../nodelabels/TestNodeAttributesManager.java | 99 ++++++++++
15 files changed, 718 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 730fb25..9d675fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3448,9 +3448,12 @@ public class YarnConfiguration extends Configuration {
public static final String NM_NODE_LABELS_PROVIDER_CONFIG =
NM_NODE_LABELS_PREFIX + "provider";
+ public static final String NM_NODE_ATTRIBUTES_PROVIDER_CONFIG =
+ NM_NODE_ATTRIBUTES_PREFIX + "provider";
+
// whitelist names for the yarn.nodemanager.node-labels.provider
- public static final String CONFIG_NODE_LABELS_PROVIDER = "config";
- public static final String SCRIPT_NODE_LABELS_PROVIDER = "script";
+ public static final String CONFIG_NODE_DESCRIPTOR_PROVIDER = "config";
+ public static final String SCRIPT_NODE_DESCRIPTOR_PROVIDER = "script";
private static final String NM_NODE_LABELS_PROVIDER_PREFIX =
NM_NODE_LABELS_PREFIX + "provider.";
@@ -3482,6 +3485,9 @@ public class YarnConfiguration extends Configuration {
public static final String NM_PROVIDER_CONFIGURED_NODE_PARTITION =
NM_NODE_LABELS_PROVIDER_PREFIX + "configured-node-partition";
+ public static final String NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES =
+ NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "configured-node-attributes";
+
private static final String RM_NODE_LABELS_PREFIX = RM_PREFIX
+ "node-labels.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
index effda9b..ffa33cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -35,15 +35,18 @@ public abstract class NodeAttributesManager extends AbstractService {
/**
* To completely replace the mappings for a given node with the new Set of
- * Attributes. If the mapping contains an attribute whose type does not match
- * a previously existing Attribute under the same prefix (name space) then
- * exception is thrown. Key would be name of the node and value would be set
- * of Attributes to be mapped.
+ * Attributes which are under a given prefix. If the mapping contains an
+ * attribute whose type does not match a previously existing Attribute
+ * under the same prefix (name space) then exception is thrown.
+ * Key would be name of the node and value would be set of Attributes to
+ * be mapped. If the prefix is null, then all node attributes will be
+ * replaced regardless of what prefix they have.
*
- * @param nodeAttributeMapping
- * @throws IOException
+ * @param prefix node attribute prefix
+ * @param nodeAttributeMapping host name to a set of node attributes mapping
+ * @throws IOException if failed to replace attributes
*/
- public abstract void replaceNodeAttributes(
+ public abstract void replaceNodeAttributes(String prefix,
Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
index fdfd0ce..93a27a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -23,6 +23,7 @@ import org.apache.hadoop.yarn.api.records.NodeAttribute;
import java.io.IOException;
import java.util.Set;
import java.util.regex.Pattern;
+import java.util.stream.Collectors;
/**
* Utility class for all NodeLabel and NodeAttribute operations.
@@ -125,4 +126,22 @@ public final class NodeLabelUtil {
}
}
}
+
+ /**
+ * Filter a set of node attributes by a given prefix. Returns a filtered
+ * set of node attributes whose prefix equals the given prefix.
+ * If the prefix is null or empty, then the original set is returned.
+ * @param attributeSet node attribute set
+ * @param prefix node attribute prefix
+ * @return a filtered set of node attributes
+ */
+ public static Set<NodeAttribute> filterAttributesByPrefix(
+ Set<NodeAttribute> attributeSet, String prefix) {
+ if (Strings.isNullOrEmpty(prefix)) {
+ return attributeSet;
+ }
+ return attributeSet.stream().filter(
+ nodeAttribute -> prefix.equals(nodeAttribute.getAttributePrefix()))
+ .collect(Collectors.toSet());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2931982..18be96b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2857,6 +2857,20 @@
<!-- Distributed Node Attributes Configuration -->
<property>
<description>
+ This property determines which provider will be plugged by the
+ node manager to collect node-attributes. Administrators can
+ configure "config", "script" or the class name of the provider.
+ Configured class needs to extend
+ org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeAttributesProvider.
+ If "config" is configured, then "ConfigurationNodeLabelsProvider" and if
+ "script" is configured, then "ScriptBasedNodeAttributesProvider"
+ will be used.
+ </description>
+ <name>yarn.nodemanager.node-attributes.provider</name>
+ </property>
+
+ <property>
+ <description>
The node attribute script NM runs to collect node attributes.
Script output Line starting with "NODE_ATTRIBUTE:" will be
considered as a record of node attribute, attribute name, type
@@ -2894,6 +2908,16 @@
<property>
<description>
+ When "yarn.nodemanager.node-attributes.provider" is configured with
+ "config" then ConfigurationNodeAttributesProvider fetches node attributes
+ from this parameter.
+ </description>
+ <name>yarn.nodemanager.node-attributes.provider.configured-node-attributes</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
Timeout in seconds for YARN node graceful decommission.
This is the maximal time to wait for running containers and applications to complete
before transition a DECOMMISSIONING node into DECOMMISSIONED.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 2748a8f..7358d9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -64,6 +64,9 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ConfigurationNodeLabelsProvider;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ScriptBasedNodeLabelsProvider;
+import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ScriptBasedNodeAttributesProvider;
+import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeAttributesProvider;
+import org.apache.hadoop.yarn.server.nodemanager.nodelabels.ConfigurationNodeAttributesProvider;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMLeveldbStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
@@ -121,6 +124,7 @@ public class NodeManager extends CompositeService
private ApplicationACLsManager aclsManager;
private NodeHealthCheckerService nodeHealthChecker;
private NodeLabelsProvider nodeLabelsProvider;
+ private NodeAttributesProvider nodeAttributesProvider;
private LocalDirsHandlerService dirsHandler;
private Context context;
private AsyncDispatcher dispatcher;
@@ -159,14 +163,45 @@ public class NodeManager extends CompositeService
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker,
- metrics, nodeLabelsProvider);
+ metrics);
}
- protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- NodeLabelsProvider nodeLabelsProvider) {
- return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker,
- metrics, nodeLabelsProvider);
+ protected NodeAttributesProvider createNodeAttributesProvider(
+ Configuration conf) throws IOException {
+ NodeAttributesProvider attributesProvider = null;
+ String providerString =
+ conf.get(YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_CONFIG, null);
+ if (providerString == null || providerString.trim().length() == 0) {
+ return attributesProvider;
+ }
+ switch (providerString.trim().toLowerCase()) {
+ case YarnConfiguration.CONFIG_NODE_DESCRIPTOR_PROVIDER:
+ attributesProvider = new ConfigurationNodeAttributesProvider();
+ break;
+ case YarnConfiguration.SCRIPT_NODE_DESCRIPTOR_PROVIDER:
+ attributesProvider = new ScriptBasedNodeAttributesProvider();
+ break;
+ default:
+ try {
+ Class<? extends NodeAttributesProvider> labelsProviderClass =
+ conf.getClass(YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_CONFIG,
+ null, NodeAttributesProvider.class);
+ attributesProvider = labelsProviderClass.newInstance();
+ } catch (InstantiationException | IllegalAccessException
+ | RuntimeException e) {
+ LOG.error("Failed to create NodeAttributesProvider"
+ + " based on Configuration", e);
+ throw new IOException(
+ "Failed to create NodeAttributesProvider : "
+ + e.getMessage(), e);
+ }
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Distributed Node Attributes is enabled"
+ + " with provider class as : "
+ + attributesProvider.getClass().toString());
+ }
+ return attributesProvider;
}
protected NodeLabelsProvider createNodeLabelsProvider(Configuration conf)
@@ -179,10 +214,10 @@ public class NodeManager extends CompositeService
return provider;
}
switch (providerString.trim().toLowerCase()) {
- case YarnConfiguration.CONFIG_NODE_LABELS_PROVIDER:
+ case YarnConfiguration.CONFIG_NODE_DESCRIPTOR_PROVIDER:
provider = new ConfigurationNodeLabelsProvider();
break;
- case YarnConfiguration.SCRIPT_NODE_LABELS_PROVIDER:
+ case YarnConfiguration.SCRIPT_NODE_DESCRIPTOR_PROVIDER:
provider = new ScriptBasedNodeLabelsProvider();
break;
default:
@@ -404,16 +439,19 @@ public class NodeManager extends CompositeService
((NMContext)context).setContainerExecutor(exec);
((NMContext)context).setDeletionService(del);
- nodeLabelsProvider = createNodeLabelsProvider(conf);
+ nodeStatusUpdater =
+ createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
- if (null == nodeLabelsProvider) {
- nodeStatusUpdater =
- createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
- } else {
+ nodeLabelsProvider = createNodeLabelsProvider(conf);
+ if (nodeLabelsProvider != null) {
addIfService(nodeLabelsProvider);
- nodeStatusUpdater =
- createNodeStatusUpdater(context, dispatcher, nodeHealthChecker,
- nodeLabelsProvider);
+ nodeStatusUpdater.setNodeLabelsProvider(nodeLabelsProvider);
+ }
+
+ nodeAttributesProvider = createNodeAttributesProvider(conf);
+ if (nodeAttributesProvider != null) {
+ addIfService(nodeAttributesProvider);
+ nodeStatusUpdater.setNodeAttributesProvider(nodeAttributesProvider);
}
nodeResourceMonitor = createNodeResourceMonitor();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
index 08892d2..142cbbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeAttributesProvider;
+import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
public interface NodeStatusUpdater extends Service {
@@ -59,4 +61,16 @@ public interface NodeStatusUpdater extends Service {
* @param ex exception that makes the node unhealthy
*/
void reportException(Exception ex);
+
+ /**
+ * Sets a node attributes provider to node manager.
+ * @param provider
+ */
+ void setNodeAttributesProvider(NodeAttributesProvider provider);
+
+ /**
+ * Sets a node labels provider to the node manager.
+ * @param provider
+ */
+ void setNodeLabelsProvider(NodeLabelsProvider provider);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 2615beb..517c6b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -85,6 +86,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeAttributesProvider;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
@@ -152,21 +154,16 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
Set<ContainerId> pendingContainersToRemove = new HashSet<ContainerId>();
private NMNodeLabelsHandler nodeLabelsHandler;
- private final NodeLabelsProvider nodeLabelsProvider;
+ private NMNodeAttributesHandler nodeAttributesHandler;
+ private NodeLabelsProvider nodeLabelsProvider;
+ private NodeAttributesProvider nodeAttributesProvider;
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
- this(context, dispatcher, healthChecker, metrics, null);
- }
-
- public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- NodeLabelsProvider nodeLabelsProvider) {
super(NodeStatusUpdaterImpl.class.getName());
this.healthChecker = healthChecker;
this.context = context;
this.dispatcher = dispatcher;
- this.nodeLabelsProvider = nodeLabelsProvider;
this.metrics = metrics;
this.recentlyStoppedContainers = new LinkedHashMap<ContainerId, Long>();
this.pendingCompletedContainers =
@@ -176,6 +173,16 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
@Override
+ public void setNodeAttributesProvider(NodeAttributesProvider provider) {
+ this.nodeAttributesProvider = provider;
+ }
+
+ @Override
+ public void setNodeLabelsProvider(NodeLabelsProvider provider) {
+ this.nodeLabelsProvider = provider;
+ }
+
+ @Override
protected void serviceInit(Configuration conf) throws Exception {
this.totalResource = NodeManagerHardwareUtils.getNodeResources(conf);
long memoryMb = totalResource.getMemorySize();
@@ -214,7 +221,11 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
YarnConfiguration.NM_RESOURCEMANAGER_MINIMUM_VERSION,
YarnConfiguration.DEFAULT_NM_RESOURCEMANAGER_MINIMUM_VERSION);
- nodeLabelsHandler = createNMNodeLabelsHandler(nodeLabelsProvider);
+ nodeLabelsHandler =
+ createNMNodeLabelsHandler(nodeLabelsProvider);
+ nodeAttributesHandler =
+ createNMNodeAttributesHandler(nodeAttributesProvider);
+
// Default duration to track stopped containers on nodemanager is 10Min.
// This should not be assigned very large value as it will remember all the
// containers stopped during that time.
@@ -856,6 +867,43 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
}
+ /**
+ * Returns a handler based on the configured node attributes provider.
+ * returns null if no provider is configured.
+ * @param provider
+ * @return attributes handler
+ */
+ private NMNodeAttributesHandler createNMNodeAttributesHandler(
+ NodeAttributesProvider provider) {
+ return provider == null ? null :
+ new NMDistributedNodeAttributesHandler(nodeAttributesProvider);
+ }
+
+ private interface NMNodeAttributesHandler {
+
+ /**
+ * @return the node attributes of this node manager.
+ */
+ Set<NodeAttribute> getNodeAttributesForHeartbeat();
+ }
+
+ private static class NMDistributedNodeAttributesHandler
+ implements NMNodeAttributesHandler {
+
+ private final NodeAttributesProvider attributesProvider;
+
+ protected NMDistributedNodeAttributesHandler(
+ NodeAttributesProvider provider) {
+ this.attributesProvider = provider;
+ }
+
+ @Override
+ public Set<NodeAttribute> getNodeAttributesForHeartbeat() {
+ return attributesProvider.getDescriptors();
+ }
+ }
+
+
private static interface NMNodeLabelsHandler {
/**
* validates nodeLabels From Provider and returns it to the caller. Also
@@ -1071,6 +1119,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
NodeHeartbeatResponse response = null;
Set<NodeLabel> nodeLabelsForHeartbeat =
nodeLabelsHandler.getNodeLabelsForHeartbeat();
+ Set<NodeAttribute> nodeAttributesForHeartbeat =
+ nodeAttributesHandler == null ? null :
+ nodeAttributesHandler.getNodeAttributesForHeartbeat();
NodeStatus nodeStatus = getNodeStatus(lastHeartbeatID);
NodeHeartbeatRequest request =
NodeHeartbeatRequest.newInstance(nodeStatus,
@@ -1079,6 +1130,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdaterImpl.this.context
.getNMTokenSecretManager().getCurrentKey(),
nodeLabelsForHeartbeat,
+ nodeAttributesForHeartbeat,
NodeStatusUpdaterImpl.this.context
.getRegisteringCollectors());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
new file mode 100644
index 0000000..74341eb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeAttributesProvider.java
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.TimerTask;
+import java.util.Set;
+
+/**
+ * Configuration based node attributes provider.
+ */
+public class ConfigurationNodeAttributesProvider
+ extends NodeAttributesProvider {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConfigurationNodeAttributesProvider.class);
+
+ public ConfigurationNodeAttributesProvider() {
+ super("Configuration Based Node Attributes Provider");
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ long taskInterval = conf.getLong(YarnConfiguration
+ .NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS,
+ YarnConfiguration
+ .DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS);
+ this.setIntervalTime(taskInterval);
+ super.serviceInit(conf);
+ }
+
+ private void updateNodeAttributesFromConfig(Configuration conf)
+ throws IOException {
+ String configuredNodeAttributes = conf.get(
+ YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES, null);
+ setDescriptors(parseAttributes(configuredNodeAttributes));
+ }
+
+ // TODO parse attributes from configuration
+ @VisibleForTesting
+ public Set<NodeAttribute> parseAttributes(String config)
+ throws IOException {
+ return new HashSet<>();
+ }
+
+ private class ConfigurationMonitorTimerTask extends TimerTask {
+ @Override
+ public void run() {
+ try {
+ updateNodeAttributesFromConfig(new YarnConfiguration());
+ } catch (Exception e) {
+ LOG.error("Failed to update node attributes from "
+ + YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_ATTRIBUTES, e);
+ }
+ }
+ }
+
+ @Override
+ protected void cleanUp() throws Exception {
+ // Nothing to cleanup
+ }
+
+ @Override
+ public TimerTask createTimerTask() {
+ return new ConfigurationMonitorTimerTask();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
index b31215b..b2c2f6e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManager.java
@@ -160,7 +160,7 @@ public class TestNodeManager {
// With valid whitelisted configurations
conf.set(YarnConfiguration.NM_NODE_LABELS_PROVIDER_CONFIG,
- YarnConfiguration.CONFIG_NODE_LABELS_PROVIDER);
+ YarnConfiguration.CONFIG_NODE_DESCRIPTOR_PROVIDER);
labelsProviderService = nodeManager.createNodeLabelsProvider(conf);
Assert.assertNotNull("LabelsProviderService should be initialized When "
+ "node labels provider class is configured", labelsProviderService);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
index 7ef23cb..3e2d963 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
@@ -225,11 +225,10 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- NodeLabelsProvider labelsProvider) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker,
- metrics, labelsProvider) {
+ metrics) {
@Override
protected ResourceTracker getRMClient() {
return resourceTracker;
@@ -325,11 +324,10 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
@Override
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- NodeLabelsProvider labelsProvider) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker,
- metrics, labelsProvider) {
+ metrics) {
@Override
protected ResourceTracker getRMClient() {
return resourceTracker;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
new file mode 100644
index 0000000..54cc8f0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
@@ -0,0 +1,185 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.BeforeClass;
+import org.junit.Before;
+import org.junit.AfterClass;
+import org.junit.After;
+import org.junit.Test;
+import org.junit.Assert;
+
+import org.mockito.Mockito;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.ArrayList;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Test class for node configuration node attributes provider.
+ */
+public class TestConfigurationNodeAttributesProvider {
+
+ private static File testRootDir = new File("target",
+ TestConfigurationNodeAttributesProvider.class.getName() + "-localDir")
+ .getAbsoluteFile();
+
+ private ConfigurationNodeAttributesProvider nodeAttributesProvider;
+
+ @BeforeClass
+ public static void create() {
+ testRootDir.mkdirs();
+ }
+
+ @Before
+ public void setup() {
+ nodeAttributesProvider = new ConfigurationNodeAttributesProvider();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (nodeAttributesProvider != null) {
+ nodeAttributesProvider.close();
+ nodeAttributesProvider.stop();
+ }
+ }
+
+ @AfterClass
+ public static void remove() throws Exception {
+ if (testRootDir.exists()) {
+ FileContext.getLocalFSFileContext()
+ .delete(new Path(testRootDir.getAbsolutePath()), true);
+ }
+ }
+
+ @Test(timeout=30000L)
+ public void testNodeAttributesFetchInterval()
+ throws IOException, InterruptedException {
+ Set<NodeAttribute> expectedAttributes1 = new HashSet<>();
+ expectedAttributes1.add(NodeAttribute
+ .newInstance("test.io", "host",
+ NodeAttributeType.STRING, "host1"));
+
+ Configuration conf = new Configuration();
+ // Set fetch interval to 1s for testing
+ conf.setLong(
+ YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS, 1000);
+ ConfigurationNodeAttributesProvider spyProvider =
+ Mockito.spy(nodeAttributesProvider);
+ Mockito.when(spyProvider.parseAttributes(Mockito.anyString()))
+ .thenReturn(expectedAttributes1);
+
+ spyProvider.init(conf);
+ spyProvider.start();
+
+ // Verify init value is honored.
+ Assert.assertEquals(expectedAttributes1, spyProvider.getDescriptors());
+
+ // Configuration provider provides a different set of attributes.
+ Set<NodeAttribute> expectedAttributes2 = new HashSet<>();
+ expectedAttributes2.add(NodeAttribute
+ .newInstance("test.io", "os",
+ NodeAttributeType.STRING, "windows"));
+ Mockito.when(spyProvider.parseAttributes(Mockito.anyString()))
+ .thenReturn(expectedAttributes2);
+
+ // Since we set fetch interval to 1s, it needs to wait for 1s until
+ // the updated attributes is updated to the provider. So we are expecting
+ // to see some old values for a short window.
+ ArrayList<String> keysMet = new ArrayList<>();
+ int numOfOldValue = 0;
+ int numOfNewValue = 0;
+ // Run 5 times in 500ms interval
+ int times=5;
+ while(times>0) {
+ Set<NodeAttribute> current = spyProvider.getDescriptors();
+ Assert.assertEquals(1, current.size());
+ String attributeName = current.iterator().next().getAttributeName();
+ if ("host".equals(attributeName)){
+ numOfOldValue++;
+ } else if ("os".equals(attributeName)) {
+ numOfNewValue++;
+ }
+ Thread.sleep(500);
+ times--;
+ }
+ // We should either see the old value or the new value.
+ Assert.assertEquals(5, numOfNewValue + numOfOldValue);
+ // Both values should be more than 0.
+ Assert.assertTrue(numOfOldValue > 0);
+ Assert.assertTrue(numOfNewValue > 0);
+ }
+
+ @Test
+ public void testDisableFetchNodeAttributes() throws IOException,
+ InterruptedException {
+ Set<NodeAttribute> expectedAttributes1 = new HashSet<>();
+ expectedAttributes1.add(NodeAttribute
+ .newInstance("test.io", "host",
+ NodeAttributeType.STRING, "host1"));
+
+ Configuration conf = new Configuration();
+ // Set fetch interval to -1 to disable refresh.
+ conf.setLong(
+ YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS, -1);
+ ConfigurationNodeAttributesProvider spyProvider =
+ Mockito.spy(nodeAttributesProvider);
+ Mockito.when(spyProvider.parseAttributes(Mockito.anyString()))
+ .thenReturn(expectedAttributes1);
+ spyProvider.init(conf);
+ spyProvider.start();
+
+ Assert.assertEquals(expectedAttributes1,
+ spyProvider.getDescriptors());
+
+ // The configuration added another attribute,
+ // as we disabled the fetch interval, this value cannot be
+ // updated to the provider.
+ Set<NodeAttribute> expectedAttributes2 = new HashSet<>();
+ expectedAttributes2.add(NodeAttribute
+ .newInstance("test.io", "os",
+ NodeAttributeType.STRING, "windows"));
+ Mockito.when(spyProvider.parseAttributes(Mockito.anyString()))
+ .thenReturn(expectedAttributes2);
+
+ // Wait a few seconds until we get the value update, expecting a failure.
+ try {
+ GenericTestUtils.waitFor(() -> {
+ Set<NodeAttribute> attributes = spyProvider.getDescriptors();
+ return "os".equalsIgnoreCase(attributes
+ .iterator().next().getAttributeName());
+ }, 500, 1000);
+ } catch (Exception e) {
+ // Make sure we get the timeout exception.
+ Assert.assertTrue(e instanceof TimeoutException);
+ return;
+ }
+
+ Assert.fail("Expecting a failure in previous check!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index e997192..8a1a9a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -33,6 +33,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.collections.CollectionUtils;
+import com.google.common.collect.ImmutableMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -51,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -646,6 +648,34 @@ public class ResourceTrackerService extends AbstractService implements
this.rmContext.getNodeManagerQueueLimitCalculator()
.createContainerQueuingLimit());
}
+
+ // 8. Get node's attributes and update node-to-attributes mapping
+ // in RMNodeAttributeManager.
+ Set<NodeAttribute> nodeAttributes = request.getNodeAttributes();
+ if (nodeAttributes != null && !nodeAttributes.isEmpty()) {
+ nodeAttributes.forEach(nodeAttribute ->
+ LOG.debug(nodeId.toString() + " ATTRIBUTE : "
+ + nodeAttribute.toString()));
+
+ // Validate attributes
+ if (!nodeAttributes.stream().allMatch(
+ nodeAttribute -> NodeAttribute.PREFIX_DISTRIBUTED
+ .equals(nodeAttribute.getAttributePrefix()))) {
+ // All attributes must be in same prefix: nm.yarn.io.
+ // Since we have the checks in NM to make sure attributes reported
+ // in HB are with correct prefix, so it should not reach here.
+ LOG.warn("Reject invalid node attributes from host: "
+ + nodeId.toString() + ", attributes in HB must have prefix "
+ + NodeAttribute.PREFIX_DISTRIBUTED);
+ } else {
+ // Replace all distributed node attributes associated with this host
+ // with the new reported attributes in node attribute manager.
+ this.rmContext.getNodeAttributesManager()
+ .replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
+ ImmutableMap.of(nodeId.getHost(), nodeAttributes));
+ }
+ }
+
return nodeHeartBeatResponse;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index a902ac6..04d74a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -33,6 +33,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -126,7 +127,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void internalUpdateAttributesOnNodes(
Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
AttributeMappingOperationType op,
- Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded) {
+ Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded,
+ String attributePrefix) {
try {
writeLock.lock();
@@ -156,8 +158,9 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
break;
case REPLACE:
clusterAttributes.putAll(newAttributesToBeAdded);
- replaceNodeToAttribute(nodeHost, node.getAttributes(), attributes);
- node.replaceAttributes(attributes);
+ replaceNodeToAttribute(nodeHost, attributePrefix,
+ node.getAttributes(), attributes);
+ node.replaceAttributes(attributes, attributePrefix);
break;
default:
break;
@@ -199,15 +202,23 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void addNodeToAttribute(String nodeHost,
Map<NodeAttribute, AttributeValue> attributeMappings) {
for (NodeAttribute attribute : attributeMappings.keySet()) {
- clusterAttributes.get(attribute).addNode(nodeHost);
+ RMNodeAttribute rmNodeAttribute = clusterAttributes.get(attribute);
+ if (rmNodeAttribute != null) {
+ rmNodeAttribute.addNode(nodeHost);
+ } else {
+ clusterAttributes.put(attribute, new RMNodeAttribute(attribute));
+ }
}
}
- private void replaceNodeToAttribute(String nodeHost,
+ private void replaceNodeToAttribute(String nodeHost, String prefix,
Map<NodeAttribute, AttributeValue> oldAttributeMappings,
Map<NodeAttribute, AttributeValue> newAttributeMappings) {
if (oldAttributeMappings != null) {
- removeNodeFromAttributes(nodeHost, oldAttributeMappings.keySet());
+ Set<NodeAttribute> toRemoveAttributes =
+ NodeLabelUtil.filterAttributesByPrefix(
+ oldAttributeMappings.keySet(), prefix);
+ removeNodeFromAttributes(nodeHost, toRemoveAttributes);
}
addNodeToAttribute(nodeHost, newAttributeMappings);
}
@@ -432,8 +443,19 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
}
public void replaceAttributes(
- Map<NodeAttribute, AttributeValue> attributesMapping) {
- this.attributes.clear();
+ Map<NodeAttribute, AttributeValue> attributesMapping, String prefix) {
+ if (Strings.isNullOrEmpty(prefix)) {
+ this.attributes.clear();
+ } else {
+ Iterator<Entry<NodeAttribute, AttributeValue>> it =
+ this.attributes.entrySet().iterator();
+ while (it.hasNext()) {
+ Entry<NodeAttribute, AttributeValue> current = it.next();
+ if (prefix.equals(current.getKey().getAttributePrefix())) {
+ it.remove();
+ }
+ }
+ }
this.attributes.putAll(attributesMapping);
}
@@ -506,9 +528,10 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
}
@Override
- public void replaceNodeAttributes(
+ public void replaceNodeAttributes(String prefix,
Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException {
- processMapping(nodeAttributeMapping, AttributeMappingOperationType.REPLACE);
+ processMapping(nodeAttributeMapping,
+ AttributeMappingOperationType.REPLACE, prefix);
}
@Override
@@ -526,12 +549,19 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void processMapping(
Map<String, Set<NodeAttribute>> nodeAttributeMapping,
AttributeMappingOperationType mappingType) throws IOException {
+ processMapping(nodeAttributeMapping, mappingType, null);
+ }
+
+ private void processMapping(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping,
+ AttributeMappingOperationType mappingType, String attributePrefix)
+ throws IOException {
Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded =
new HashMap<>();
Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
validate(nodeAttributeMapping, newAttributesToBeAdded, false);
internalUpdateAttributesOnNodes(validMapping, mappingType,
- newAttributesToBeAdded);
+ newAttributesToBeAdded, attributePrefix);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index fa0f5fd..a29e8a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -37,6 +37,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.HashSet;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import javax.xml.parsers.DocumentBuilderFactory;
@@ -64,12 +65,16 @@ import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.nodelabels.AttributeValue;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelTestBase;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
@@ -818,6 +823,79 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
}
@Test
+ public void testNodeHeartbeatWithNodeAttributes() throws Exception {
+ writeToHostsFile("host2");
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.RM_NODES_INCLUDE_FILE_PATH,
+ hostFile.getAbsolutePath());
+ rm = new MockRM(conf);
+ rm.start();
+
+ // Register to RM
+ ResourceTrackerService resourceTrackerService =
+ rm.getResourceTrackerService();
+ RegisterNodeManagerRequest registerReq =
+ Records.newRecord(RegisterNodeManagerRequest.class);
+ NodeId nodeId = NodeId.newInstance("host2", 1234);
+ Resource capability = BuilderUtils.newResource(1024, 1);
+ registerReq.setResource(capability);
+ registerReq.setNodeId(nodeId);
+ registerReq.setHttpPort(1234);
+ registerReq.setNMVersion(YarnVersionInfo.getVersion());
+ RegisterNodeManagerResponse registerResponse =
+ resourceTrackerService.registerNodeManager(registerReq);
+
+ Set<NodeAttribute> nodeAttributes = new HashSet<>();
+ nodeAttributes.add(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_DISTRIBUTED, "host",
+ NodeAttributeType.STRING, "host2"));
+
+ // Set node attributes in HB.
+ NodeHeartbeatRequest heartbeatReq =
+ Records.newRecord(NodeHeartbeatRequest.class);
+ NodeStatus nodeStatusObject = getNodeStatusObject(nodeId);
+ int responseId = nodeStatusObject.getResponseId();
+ heartbeatReq.setNodeStatus(nodeStatusObject);
+ heartbeatReq.setLastKnownNMTokenMasterKey(registerResponse
+ .getNMTokenMasterKey());
+ heartbeatReq.setLastKnownContainerTokenMasterKey(registerResponse
+ .getContainerTokenMasterKey());
+ heartbeatReq.setNodeAttributes(nodeAttributes);
+ resourceTrackerService.nodeHeartbeat(heartbeatReq);
+
+ // Ensure RM gets correct node attributes update.
+ NodeAttributesManager attributeManager =
+ rm.getRMContext().getNodeAttributesManager();
+ Map<NodeAttribute, AttributeValue> attrs = attributeManager
+ .getAttributesForNode(nodeId.getHost());
+ Assert.assertEquals(1, attrs.size());
+ NodeAttribute na = attrs.keySet().iterator().next();
+ Assert.assertEquals("host", na.getAttributeName());
+ Assert.assertEquals("host2", na.getAttributeValue());
+ Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType());
+
+
+ // Send another HB to RM with updated node atrribute
+ nodeAttributes.clear();
+ nodeAttributes.add(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_DISTRIBUTED, "host",
+ NodeAttributeType.STRING, "host3"));
+ nodeStatusObject = getNodeStatusObject(nodeId);
+ nodeStatusObject.setResponseId(++responseId);
+ heartbeatReq.setNodeStatus(nodeStatusObject);
+ heartbeatReq.setNodeAttributes(nodeAttributes);
+ resourceTrackerService.nodeHeartbeat(heartbeatReq);
+
+ // Make sure RM gets the updated attribute
+ attrs = attributeManager.getAttributesForNode(nodeId.getHost());
+ Assert.assertEquals(1, attrs.size());
+ na = attrs.keySet().iterator().next();
+ Assert.assertEquals("host", na.getAttributeName());
+ Assert.assertEquals("host3", na.getAttributeValue());
+ Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType());
+ }
+
+ @Test
public void testNodeHeartBeatWithInvalidLabels() throws Exception {
writeToHostsFile("host2");
Configuration conf = new Configuration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5bd044e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
index b639a74..07968d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.nodelabels.AttributeValue;
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
import org.junit.Test;
import org.junit.Before;
import org.junit.After;
@@ -255,4 +257,101 @@ public class TestNodeAttributesManager {
.getClusterNodeAttributes(Sets.newHashSet(PREFIXES[0]));
Assert.assertEquals(2, allAttributesPerPrefix.size());
}
+
+ @Test
+ public void testReplaceNodeAttributes() throws IOException {
+ Map<String, Set<NodeAttribute>> toAddAttributes = new HashMap<>();
+ Map<String, Set<NodeAttribute>> toReplaceMap = new HashMap<>();
+ Map<NodeAttribute, AttributeValue> nodeAttributes;
+ Set<NodeAttribute> filteredAttributes;
+ Set<NodeAttribute> clusterAttributes;
+
+ // Add 3 attributes to host1
+ // yarn.test1.io/A1=host1_v1_1
+ // yarn.test1.io/A2=host1_v1_2
+ // yarn.test1.io/A3=host1_v1_3
+ toAddAttributes.put(HOSTNAMES[0],
+ createAttributesForTest(PREFIXES[0], 3, "A", "host1_v1"));
+
+ attributesManager.addNodeAttributes(toAddAttributes);
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(3, nodeAttributes.size());
+
+ // Add 10 distributed node attributes to host1
+ // nn.yarn.io/dist-node-attribute1=dist_v1_1
+ // nn.yarn.io/dist-node-attribute2=dist_v1_2
+ // ...
+ // nn.yarn.io/dist-node-attribute10=dist_v1_10
+ toAddAttributes.clear();
+ toAddAttributes.put(HOSTNAMES[0],
+ createAttributesForTest(NodeAttribute.PREFIX_DISTRIBUTED,
+ 10, "dist-node-attribute", "dist_v1"));
+ attributesManager.addNodeAttributes(toAddAttributes);
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(13, nodeAttributes.size());
+ clusterAttributes = attributesManager.getClusterNodeAttributes(
+ Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED, PREFIXES[0]));
+ Assert.assertEquals(13, clusterAttributes.size());
+
+ // Replace by prefix
+ // Same distributed attributes names, but different values.
+ Set<NodeAttribute> toReplaceAttributes =
+ createAttributesForTest(NodeAttribute.PREFIX_DISTRIBUTED, 5,
+ "dist-node-attribute", "dist_v2");
+
+ attributesManager.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
+ ImmutableMap.of(HOSTNAMES[0], toReplaceAttributes));
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(8, nodeAttributes.size());
+ clusterAttributes = attributesManager.getClusterNodeAttributes(
+ Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED, PREFIXES[0]));
+ Assert.assertEquals(8, clusterAttributes.size());
+
+ // Now we have 5 distributed attributes
+ filteredAttributes = NodeLabelUtil.filterAttributesByPrefix(
+ nodeAttributes.keySet(), NodeAttribute.PREFIX_DISTRIBUTED);
+ Assert.assertEquals(5, filteredAttributes.size());
+ // Values are updated to have prefix dist_v2
+ Assert.assertTrue(filteredAttributes.stream().allMatch(
+ nodeAttribute ->
+ nodeAttribute.getAttributeValue().startsWith("dist_v2")));
+
+ // We still have 3 yarn.test1.io attributes
+ filteredAttributes = NodeLabelUtil.filterAttributesByPrefix(
+ nodeAttributes.keySet(), PREFIXES[0]);
+ Assert.assertEquals(3, filteredAttributes.size());
+
+ // Replace with prefix
+ // Different attribute names
+ toReplaceAttributes =
+ createAttributesForTest(NodeAttribute.PREFIX_DISTRIBUTED, 1,
+ "dist-node-attribute-v2", "dist_v3");
+ attributesManager.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
+ ImmutableMap.of(HOSTNAMES[0], toReplaceAttributes));
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(4, nodeAttributes.size());
+ clusterAttributes = attributesManager.getClusterNodeAttributes(
+ Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED));
+ Assert.assertEquals(1, clusterAttributes.size());
+ NodeAttribute att = clusterAttributes.iterator().next();
+ Assert.assertEquals("dist-node-attribute-v2_0", att.getAttributeName());
+ Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED,
+ att.getAttributePrefix());
+ Assert.assertEquals("dist_v3_0", att.getAttributeValue());
+
+ // Replace all attributes
+ toReplaceMap.put(HOSTNAMES[0],
+ createAttributesForTest(PREFIXES[1], 2, "B", "B_v1"));
+ attributesManager.replaceNodeAttributes(null, toReplaceMap);
+
+ nodeAttributes = attributesManager.getAttributesForNode(HOSTNAMES[0]);
+ Assert.assertEquals(2, nodeAttributes.size());
+ clusterAttributes = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(PREFIXES[1]));
+ Assert.assertEquals(2, clusterAttributes.size());
+ clusterAttributes = attributesManager
+ .getClusterNodeAttributes(Sets.newHashSet(
+ NodeAttribute.PREFIX_DISTRIBUTED));
+ Assert.assertEquals(0, clusterAttributes.size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-15312. Undocumented
KeyProvider configuration keys. Contributed by LiXin Ge.
Posted by na...@apache.org.
HADOOP-15312. Undocumented KeyProvider configuration keys. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3fe41c65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3fe41c65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3fe41c65
Branch: refs/heads/YARN-3409
Commit: 3fe41c65d84843f817a4f9ef8999dbf862db6674
Parents: 285bbaa
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Tue Mar 27 13:33:41 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Tue Mar 27 13:33:41 2018 -0700
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fe41c65/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 9074300..49bfa28 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -2495,6 +2495,25 @@
</property>
<property>
+ <name>hadoop.security.key.default.bitlength</name>
+ <value>128</value>
+ <description>
+ The length (bits) of keys we want the KeyProvider to produce. Key length
+ defines the upper-bound on an algorithm's security, ideally, it would
+ coincide with the lower-bound on an algorithm's security.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.key.default.cipher</name>
+ <value>AES/CTR/NoPadding</value>
+ <description>
+ This indicates the algorithm that be used by KeyProvider for generating
+ key, and will be converted to CipherSuite when creating encryption zone.
+ </description>
+</property>
+
+<property>
<name>fs.har.impl.disable.cache</name>
<value>true</value>
<description>Don't cache 'har' filesystem instances.</description>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: HADOOP-14999. AliyunOSS: provide
one asynchronous multi-part based uploading mechanism. Contributed by Genmao
Yu.
Posted by na...@apache.org.
HADOOP-14999. AliyunOSS: provide one asynchronous multi-part based uploading mechanism. Contributed by Genmao Yu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6542d17e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6542d17e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6542d17e
Branch: refs/heads/YARN-3409
Commit: 6542d17ea460ec222137c4b275b13daf15d3fca3
Parents: 2216bde
Author: Sammi Chen <sa...@intel.com>
Authored: Fri Mar 30 20:23:05 2018 +0800
Committer: Sammi Chen <sa...@intel.com>
Committed: Fri Mar 30 20:23:05 2018 +0800
----------------------------------------------------------------------
.../aliyun/oss/AliyunCredentialsProvider.java | 3 +-
.../aliyun/oss/AliyunOSSBlockOutputStream.java | 206 +++++++++++++++++++
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 34 ++-
.../fs/aliyun/oss/AliyunOSSFileSystemStore.java | 173 ++++++++--------
.../fs/aliyun/oss/AliyunOSSOutputStream.java | 111 ----------
.../hadoop/fs/aliyun/oss/AliyunOSSUtils.java | 115 ++++++++---
.../apache/hadoop/fs/aliyun/oss/Constants.java | 22 +-
.../oss/TestAliyunOSSBlockOutputStream.java | 115 +++++++++++
.../fs/aliyun/oss/TestAliyunOSSInputStream.java | 10 +-
.../aliyun/oss/TestAliyunOSSOutputStream.java | 91 --------
.../contract/TestAliyunOSSContractDistCp.java | 2 +-
11 files changed, 544 insertions(+), 338 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
index b46c67a..58c14a9 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunCredentialsProvider.java
@@ -35,8 +35,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
public class AliyunCredentialsProvider implements CredentialsProvider {
private Credentials credentials = null;
- public AliyunCredentialsProvider(Configuration conf)
- throws IOException {
+ public AliyunCredentialsProvider(Configuration conf) throws IOException {
String accessKeyId;
String accessKeySecret;
String securityToken;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
new file mode 100644
index 0000000..12d551b
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSBlockOutputStream.java
@@ -0,0 +1,206 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import com.aliyun.oss.model.PartETag;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
+import org.apache.hadoop.conf.Configuration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+
+/**
+ * Asynchronous multi-part based uploading mechanism to support huge file
+ * which is larger than 5GB. Data will be buffered on local disk, then uploaded
+ * to OSS in {@link #close()} method.
+ */
+public class AliyunOSSBlockOutputStream extends OutputStream {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AliyunOSSBlockOutputStream.class);
+ private AliyunOSSFileSystemStore store;
+ private Configuration conf;
+ private boolean closed;
+ private String key;
+ private File blockFile;
+ private List<File> blockFiles = new ArrayList<>();
+ private long blockSize;
+ private int blockId = 0;
+ private long blockWritten = 0L;
+ private String uploadId = null;
+ private final List<ListenableFuture<PartETag>> partETagsFutures;
+ private final ListeningExecutorService executorService;
+ private OutputStream blockStream;
+ private final byte[] singleByte = new byte[1];
+
+ public AliyunOSSBlockOutputStream(Configuration conf,
+ AliyunOSSFileSystemStore store,
+ String key,
+ Long blockSize,
+ ExecutorService executorService) throws IOException {
+ this.store = store;
+ this.conf = conf;
+ this.key = key;
+ this.blockSize = blockSize;
+ this.blockFile = newBlockFile();
+ this.blockStream =
+ new BufferedOutputStream(new FileOutputStream(blockFile));
+ this.partETagsFutures = new ArrayList<>(2);
+ this.executorService = MoreExecutors.listeningDecorator(executorService);
+ }
+
+ private File newBlockFile() throws IOException {
+ return AliyunOSSUtils.createTmpFileForWrite(
+ String.format("oss-block-%04d-", blockId), blockSize, conf);
+ }
+
+ @Override
+ public synchronized void flush() throws IOException {
+ blockStream.flush();
+ }
+
+ @Override
+ public synchronized void close() throws IOException {
+ if (closed) {
+ return;
+ }
+
+ blockStream.flush();
+ blockStream.close();
+ if (!blockFiles.contains(blockFile)) {
+ blockFiles.add(blockFile);
+ }
+
+ try {
+ if (blockFiles.size() == 1) {
+ // just upload it directly
+ store.uploadObject(key, blockFile);
+ } else {
+ if (blockWritten > 0) {
+ ListenableFuture<PartETag> partETagFuture =
+ executorService.submit(() -> {
+ PartETag partETag = store.uploadPart(blockFile, key, uploadId,
+ blockId + 1);
+ return partETag;
+ });
+ partETagsFutures.add(partETagFuture);
+ }
+ // wait for the partial uploads to finish
+ final List<PartETag> partETags = waitForAllPartUploads();
+ if (null == partETags) {
+ throw new IOException("Failed to multipart upload to oss, abort it.");
+ }
+ store.completeMultipartUpload(key, uploadId, partETags);
+ }
+ } finally {
+ for (File tFile: blockFiles) {
+ if (tFile.exists() && !tFile.delete()) {
+ LOG.warn("Failed to delete temporary file {}", tFile);
+ }
+ }
+ closed = true;
+ }
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ singleByte[0] = (byte)b;
+ write(singleByte, 0, 1);
+ }
+
+ @Override
+ public synchronized void write(byte[] b, int off, int len)
+ throws IOException {
+ if (closed) {
+ throw new IOException("Stream closed.");
+ }
+ try {
+ blockStream.write(b, off, len);
+ blockWritten += len;
+ if (blockWritten >= blockSize) {
+ uploadCurrentPart();
+ blockWritten = 0L;
+ }
+ } finally {
+ for (File tFile: blockFiles) {
+ if (tFile.exists() && !tFile.delete()) {
+ LOG.warn("Failed to delete temporary file {}", tFile);
+ }
+ }
+ }
+ }
+
+ private void uploadCurrentPart() throws IOException {
+ blockFiles.add(blockFile);
+ blockStream.flush();
+ blockStream.close();
+ if (blockId == 0) {
+ uploadId = store.getUploadId(key);
+ }
+ ListenableFuture<PartETag> partETagFuture =
+ executorService.submit(() -> {
+ PartETag partETag = store.uploadPart(blockFile, key, uploadId,
+ blockId + 1);
+ return partETag;
+ });
+ partETagsFutures.add(partETagFuture);
+ blockFile = newBlockFile();
+ blockId++;
+ blockStream = new BufferedOutputStream(new FileOutputStream(blockFile));
+ }
+
+ /**
+ * Block awaiting all outstanding uploads to complete.
+ * @return list of results
+ * @throws IOException IO Problems
+ */
+ private List<PartETag> waitForAllPartUploads() throws IOException {
+ LOG.debug("Waiting for {} uploads to complete", partETagsFutures.size());
+ try {
+ return Futures.allAsList(partETagsFutures).get();
+ } catch (InterruptedException ie) {
+ LOG.warn("Interrupted partUpload", ie);
+ Thread.currentThread().interrupt();
+ return null;
+ } catch (ExecutionException ee) {
+ //there is no way of recovering so abort
+ //cancel all partUploads
+ LOG.debug("While waiting for upload completion", ee);
+ LOG.debug("Cancelling futures");
+ for (ListenableFuture<PartETag> future : partETagsFutures) {
+ future.cancel(true);
+ }
+ //abort multipartupload
+ store.abortMultipartUpload(key, uploadId);
+ throw new IOException("Multi-part upload with id '" + uploadId
+ + "' to " + key, ee);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index b3c63d3..93e31d5 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.util.SemaphoredDelegatingExecutor;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.intOption;
+import static org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.longOption;
import static org.apache.hadoop.fs.aliyun.oss.AliyunOSSUtils.objectRepresentsDirectory;
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
@@ -69,6 +71,7 @@ public class AliyunOSSFileSystem extends FileSystem {
private URI uri;
private String bucket;
private Path workingDir;
+ private int blockOutputActiveBlocks;
private AliyunOSSFileSystemStore store;
private int maxKeys;
private int maxReadAheadPartNumber;
@@ -125,8 +128,15 @@ public class AliyunOSSFileSystem extends FileSystem {
// this means the file is not found
}
- return new FSDataOutputStream(new AliyunOSSOutputStream(getConf(),
- store, key, progress, statistics), (Statistics)(null));
+ long uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(getConf(),
+ MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
+ return new FSDataOutputStream(
+ new AliyunOSSBlockOutputStream(getConf(),
+ store,
+ key,
+ uploadPartSize,
+ new SemaphoredDelegatingExecutor(boundedThreadPool,
+ blockOutputActiveBlocks, true)), (Statistics)(null));
}
/**
@@ -149,9 +159,8 @@ public class AliyunOSSFileSystem extends FileSystem {
throw new FileAlreadyExistsException("Not a directory: " + parent);
}
}
- return create(path, permission,
- flags.contains(CreateFlag.OVERWRITE), bufferSize,
- replication, blockSize, progress);
+ return create(path, permission, flags.contains(CreateFlag.OVERWRITE),
+ bufferSize, replication, blockSize, progress);
}
@Override
@@ -270,7 +279,7 @@ public class AliyunOSSFileSystem extends FileSystem {
}
} else if (objectRepresentsDirectory(key, meta.getContentLength())) {
return new FileStatus(0, true, 1, 0, meta.getLastModified().getTime(),
- qualifiedPath);
+ qualifiedPath);
} else {
return new FileStatus(meta.getContentLength(), false, 1,
getDefaultBlockSize(path), meta.getLastModified().getTime(),
@@ -318,6 +327,10 @@ public class AliyunOSSFileSystem extends FileSystem {
uri = java.net.URI.create(name.getScheme() + "://" + name.getAuthority());
workingDir = new Path("/user",
System.getProperty("user.name")).makeQualified(uri, null);
+ long keepAliveTime = longOption(conf,
+ KEEPALIVE_TIME_KEY, KEEPALIVE_TIME_DEFAULT, 0);
+ blockOutputActiveBlocks = intOption(conf,
+ UPLOAD_ACTIVE_BLOCKS_KEY, UPLOAD_ACTIVE_BLOCKS_DEFAULT, 1);
store = new AliyunOSSFileSystemStore();
store.initialize(name, conf, statistics);
@@ -335,7 +348,8 @@ public class AliyunOSSFileSystem extends FileSystem {
Constants.MULTIPART_DOWNLOAD_AHEAD_PART_MAX_NUM_DEFAULT);
this.boundedThreadPool = BlockingThreadPoolExecutorService.newInstance(
- threadNum, totalTasks, 60L, TimeUnit.SECONDS, "oss-read-shared");
+ threadNum, totalTasks, keepAliveTime, TimeUnit.SECONDS,
+ "oss-transfer-shared");
maxConcurrentCopyTasksPerDir = AliyunOSSUtils.intPositiveOption(conf,
Constants.MAX_CONCURRENT_COPY_TASKS_PER_DIR_KEY,
@@ -490,12 +504,12 @@ public class AliyunOSSFileSystem extends FileSystem {
if (status.isFile()) {
LOG.debug("{} is a File", qualifiedPath);
final BlockLocation[] locations = getFileBlockLocations(status,
- 0, status.getLen());
+ 0, status.getLen());
return store.singleStatusRemoteIterator(filter.accept(f) ? status : null,
- locations);
+ locations);
} else {
return store.createLocatedFileStatusIterator(key, maxKeys, this, filter,
- acceptor, recursive ? null : "/");
+ acceptor, recursive ? null : "/");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index a7f13c0..cc050c8 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.fs.aliyun.oss;
import com.aliyun.oss.ClientConfiguration;
@@ -62,8 +63,11 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.Serializable;
import java.net.URI;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
import java.util.List;
import java.util.ListIterator;
import java.util.NoSuchElementException;
@@ -83,7 +87,6 @@ public class AliyunOSSFileSystemStore {
private String bucketName;
private long uploadPartSize;
private long multipartThreshold;
- private long partSize;
private int maxKeys;
private String serverSideEncryptionAlgorithm;
@@ -143,28 +146,18 @@ public class AliyunOSSFileSystemStore {
String endPoint = conf.getTrimmed(ENDPOINT_KEY, "");
if (StringUtils.isEmpty(endPoint)) {
throw new IllegalArgumentException("Aliyun OSS endpoint should not be " +
- "null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
+ "null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
}
CredentialsProvider provider =
AliyunOSSUtils.getCredentialsProvider(conf);
ossClient = new OSSClient(endPoint, provider, clientConf);
- uploadPartSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
- MULTIPART_UPLOAD_SIZE_DEFAULT);
+ uploadPartSize = AliyunOSSUtils.getMultipartSizeProperty(conf,
+ MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
multipartThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
- partSize = conf.getLong(MULTIPART_UPLOAD_SIZE_KEY,
- MULTIPART_UPLOAD_SIZE_DEFAULT);
- if (partSize < MIN_MULTIPART_UPLOAD_PART_SIZE) {
- partSize = MIN_MULTIPART_UPLOAD_PART_SIZE;
- }
serverSideEncryptionAlgorithm =
conf.get(SERVER_SIDE_ENCRYPTION_ALGORITHM_KEY, "");
- if (uploadPartSize < 5 * 1024 * 1024) {
- LOG.warn(MULTIPART_UPLOAD_SIZE_KEY + " must be at least 5 MB");
- uploadPartSize = 5 * 1024 * 1024;
- }
-
if (multipartThreshold < 5 * 1024 * 1024) {
LOG.warn(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY + " must be at least 5 MB");
multipartThreshold = 5 * 1024 * 1024;
@@ -420,71 +413,6 @@ public class AliyunOSSFileSystemStore {
}
/**
- * Upload a file as an OSS object, using multipart upload.
- *
- * @param key object key.
- * @param file local file to upload.
- * @throws IOException if failed to upload object.
- */
- public void multipartUploadObject(String key, File file) throws IOException {
- File object = file.getAbsoluteFile();
- long dataLen = object.length();
- long realPartSize = AliyunOSSUtils.calculatePartSize(dataLen, partSize);
- int partNum = (int) (dataLen / realPartSize);
- if (dataLen % realPartSize != 0) {
- partNum += 1;
- }
-
- InitiateMultipartUploadRequest initiateMultipartUploadRequest =
- new InitiateMultipartUploadRequest(bucketName, key);
- ObjectMetadata meta = new ObjectMetadata();
- if (StringUtils.isNotEmpty(serverSideEncryptionAlgorithm)) {
- meta.setServerSideEncryption(serverSideEncryptionAlgorithm);
- }
- initiateMultipartUploadRequest.setObjectMetadata(meta);
- InitiateMultipartUploadResult initiateMultipartUploadResult =
- ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
- List<PartETag> partETags = new ArrayList<PartETag>();
- String uploadId = initiateMultipartUploadResult.getUploadId();
-
- try {
- for (int i = 0; i < partNum; i++) {
- // TODO: Optimize this, avoid opening the object multiple times
- FileInputStream fis = new FileInputStream(object);
- try {
- long skipBytes = realPartSize * i;
- AliyunOSSUtils.skipFully(fis, skipBytes);
- long size = (realPartSize < dataLen - skipBytes) ?
- realPartSize : dataLen - skipBytes;
- UploadPartRequest uploadPartRequest = new UploadPartRequest();
- uploadPartRequest.setBucketName(bucketName);
- uploadPartRequest.setKey(key);
- uploadPartRequest.setUploadId(uploadId);
- uploadPartRequest.setInputStream(fis);
- uploadPartRequest.setPartSize(size);
- uploadPartRequest.setPartNumber(i + 1);
- UploadPartResult uploadPartResult =
- ossClient.uploadPart(uploadPartRequest);
- statistics.incrementWriteOps(1);
- partETags.add(uploadPartResult.getPartETag());
- } finally {
- fis.close();
- }
- }
- CompleteMultipartUploadRequest completeMultipartUploadRequest =
- new CompleteMultipartUploadRequest(bucketName, key,
- uploadId, partETags);
- CompleteMultipartUploadResult completeMultipartUploadResult =
- ossClient.completeMultipartUpload(completeMultipartUploadRequest);
- LOG.debug(completeMultipartUploadResult.getETag());
- } catch (OSSException | ClientException e) {
- AbortMultipartUploadRequest abortMultipartUploadRequest =
- new AbortMultipartUploadRequest(bucketName, key, uploadId);
- ossClient.abortMultipartUpload(abortMultipartUploadRequest);
- }
- }
-
- /**
* list objects.
*
* @param prefix prefix.
@@ -494,7 +422,7 @@ public class AliyunOSSFileSystemStore {
* @return a list of matches.
*/
public ObjectListing listObjects(String prefix, int maxListingLength,
- String marker, boolean recursive) {
+ String marker, boolean recursive) {
String delimiter = recursive ? null : "/";
prefix = AliyunOSSUtils.maybeAddTrailingSlash(prefix);
ListObjectsRequest listRequest = new ListObjectsRequest(bucketName);
@@ -605,7 +533,7 @@ public class AliyunOSSFileSystemStore {
if (hasNext()) {
FileStatus status = batchIterator.next();
BlockLocation[] locations = fs.getFileBlockLocations(status,
- 0, status.getLen());
+ 0, status.getLen());
return new LocatedFileStatus(
status, status.isFile() ? locations : null);
} else {
@@ -626,7 +554,7 @@ public class AliyunOSSFileSystemStore {
List<FileStatus> stats = new ArrayList<>(
listing.getObjectSummaries().size() +
listing.getCommonPrefixes().size());
- for(OSSObjectSummary summary: listing.getObjectSummaries()) {
+ for (OSSObjectSummary summary : listing.getObjectSummaries()) {
String key = summary.getKey();
Path path = fs.makeQualified(new Path("/" + key));
if (filter.accept(path) && acceptor.accept(path, summary)) {
@@ -637,7 +565,7 @@ public class AliyunOSSFileSystemStore {
}
}
- for(String commonPrefix: listing.getCommonPrefixes()) {
+ for (String commonPrefix : listing.getCommonPrefixes()) {
Path path = fs.makeQualified(new Path("/" + commonPrefix));
if (filter.accept(path) && acceptor.accept(path, commonPrefix)) {
FileStatus status = new FileStatus(0, true, 1, 0, 0, path);
@@ -656,4 +584,83 @@ public class AliyunOSSFileSystemStore {
}
};
}
+
+ public PartETag uploadPart(File file, String key, String uploadId, int idx)
+ throws IOException {
+ InputStream instream = null;
+ Exception caught = null;
+ int tries = 3;
+ while (tries > 0) {
+ try {
+ instream = new FileInputStream(file);
+ UploadPartRequest uploadRequest = new UploadPartRequest();
+ uploadRequest.setBucketName(bucketName);
+ uploadRequest.setKey(key);
+ uploadRequest.setUploadId(uploadId);
+ uploadRequest.setInputStream(instream);
+ uploadRequest.setPartSize(file.length());
+ uploadRequest.setPartNumber(idx);
+ UploadPartResult uploadResult = ossClient.uploadPart(uploadRequest);
+ return uploadResult.getPartETag();
+ } catch (Exception e) {
+ LOG.debug("Failed to upload "+ file.getPath() +", " +
+ "try again.", e);
+ caught = e;
+ } finally {
+ if (instream != null) {
+ instream.close();
+ instream = null;
+ }
+ }
+ tries--;
+ }
+
+ assert (caught != null);
+ throw new IOException("Failed to upload " + file.getPath() +
+ " for 3 times.", caught);
+ }
+
+ /**
+ * Initiate multipart upload.
+ */
+ public String getUploadId(String key) {
+ InitiateMultipartUploadRequest initiateMultipartUploadRequest =
+ new InitiateMultipartUploadRequest(bucketName, key);
+ InitiateMultipartUploadResult initiateMultipartUploadResult =
+ ossClient.initiateMultipartUpload(initiateMultipartUploadRequest);
+ return initiateMultipartUploadResult.getUploadId();
+ }
+
+ /**
+ * Complete the specific multipart upload.
+ */
+ public CompleteMultipartUploadResult completeMultipartUpload(String key,
+ String uploadId, List<PartETag> partETags) {
+ Collections.sort(partETags, new PartNumberAscendComparator());
+ CompleteMultipartUploadRequest completeMultipartUploadRequest =
+ new CompleteMultipartUploadRequest(bucketName, key, uploadId,
+ partETags);
+ return ossClient.completeMultipartUpload(completeMultipartUploadRequest);
+ }
+
+ /**
+ * Abort the specific multipart upload.
+ */
+ public void abortMultipartUpload(String key, String uploadId) {
+ AbortMultipartUploadRequest request = new AbortMultipartUploadRequest(
+ bucketName, key, uploadId);
+ ossClient.abortMultipartUpload(request);
+ }
+
+ private static class PartNumberAscendComparator
+ implements Comparator<PartETag>, Serializable {
+ @Override
+ public int compare(PartETag o1, PartETag o2) {
+ if (o1.getPartNumber() > o2.getPartNumber()) {
+ return 1;
+ } else {
+ return -1;
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
deleted file mode 100644
index c75ee18..0000000
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSOutputStream.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem.Statistics;
-import org.apache.hadoop.fs.LocalDirAllocator;
-import org.apache.hadoop.util.Progressable;
-
-import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
-
-/**
- * The output stream for OSS blob system.
- * Data will be buffered on local disk, then uploaded to OSS in
- * {@link #close()} method.
- */
-public class AliyunOSSOutputStream extends OutputStream {
- public static final Log LOG = LogFactory.getLog(AliyunOSSOutputStream.class);
- private AliyunOSSFileSystemStore store;
- private final String key;
- private Statistics statistics;
- private Progressable progress;
- private long partSizeThreshold;
- private LocalDirAllocator dirAlloc;
- private boolean closed;
- private File tmpFile;
- private BufferedOutputStream backupStream;
-
- public AliyunOSSOutputStream(Configuration conf,
- AliyunOSSFileSystemStore store, String key, Progressable progress,
- Statistics statistics) throws IOException {
- this.store = store;
- this.key = key;
- // The caller cann't get any progress information
- this.progress = progress;
- this.statistics = statistics;
- partSizeThreshold = conf.getLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY,
- MIN_MULTIPART_UPLOAD_THRESHOLD_DEFAULT);
-
- if (conf.get(BUFFER_DIR_KEY) == null) {
- conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
- }
- dirAlloc = new LocalDirAllocator(BUFFER_DIR_KEY);
-
- tmpFile = dirAlloc.createTmpFileForWrite("output-",
- LocalDirAllocator.SIZE_UNKNOWN, conf);
- backupStream = new BufferedOutputStream(new FileOutputStream(tmpFile));
- closed = false;
- }
-
- @Override
- public synchronized void close() throws IOException {
- if (closed) {
- return;
- }
- closed = true;
- if (backupStream != null) {
- backupStream.close();
- }
- long dataLen = tmpFile.length();
- try {
- if (dataLen <= partSizeThreshold) {
- store.uploadObject(key, tmpFile);
- } else {
- store.multipartUploadObject(key, tmpFile);
- }
- } finally {
- if (!tmpFile.delete()) {
- LOG.warn("Can not delete file: " + tmpFile);
- }
- }
- }
-
-
-
- @Override
- public synchronized void flush() throws IOException {
- backupStream.flush();
- }
-
- @Override
- public synchronized void write(int b) throws IOException {
- backupStream.write(b);
- statistics.incrementBytesWritten(1);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
index 1a21608..2fe06c1 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSUtils.java
@@ -18,12 +18,14 @@
package org.apache.hadoop.fs.aliyun.oss;
+import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
import com.aliyun.oss.common.auth.CredentialsProvider;
+import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.security.ProviderUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -36,6 +38,7 @@ import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
final public class AliyunOSSUtils {
private static final Logger LOG =
LoggerFactory.getLogger(AliyunOSSUtils.class);
+ private static LocalDirAllocator directoryAllocator;
private AliyunOSSUtils() {
}
@@ -75,31 +78,6 @@ final public class AliyunOSSUtils {
}
/**
- * Skip the requested number of bytes or fail if there are no enough bytes
- * left. This allows for the possibility that {@link InputStream#skip(long)}
- * may not skip as many bytes as requested (most likely because of reaching
- * EOF).
- *
- * @param is the input stream to skip.
- * @param n the number of bytes to skip.
- * @throws IOException thrown when skipped less number of bytes.
- */
- public static void skipFully(InputStream is, long n) throws IOException {
- long total = 0;
- long cur = 0;
-
- do {
- cur = is.skip(n - total);
- total += cur;
- } while((total < n) && (cur > 0));
-
- if (total < n) {
- throw new IOException("Failed to skip " + n + " bytes, possibly due " +
- "to EOF.");
- }
- }
-
- /**
* Calculate a proper size of multipart piece. If <code>minPartSize</code>
* is too small, the number of multipart pieces may exceed the limit of
* {@link Constants#MULTIPART_UPLOAD_PART_NUM_LIMIT}.
@@ -126,7 +104,7 @@ final public class AliyunOSSUtils {
throws IOException {
CredentialsProvider credentials;
- String className = conf.getTrimmed(ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY);
+ String className = conf.getTrimmed(CREDENTIALS_PROVIDER_KEY);
if (StringUtils.isEmpty(className)) {
Configuration newConf =
ProviderUtils.excludeIncompatibleCredentialProviders(conf,
@@ -151,7 +129,7 @@ final public class AliyunOSSUtils {
throw new IOException(String.format("%s constructor exception. A " +
"class specified in %s must provide an accessible constructor " +
"accepting URI and Configuration, or an accessible default " +
- "constructor.", className, ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY),
+ "constructor.", className, CREDENTIALS_PROVIDER_KEY),
e);
} catch (ReflectiveOperationException | IllegalArgumentException e) {
throw new IOException(className + " instantiation exception.", e);
@@ -188,4 +166,85 @@ final public class AliyunOSSUtils {
final long size) {
return StringUtils.isNotEmpty(name) && name.endsWith("/") && size == 0L;
}
+
+ /**
+ * Demand create the directory allocator, then create a temporary file.
+ * @param path prefix for the temporary file
+ * @param size the size of the file that is going to be written
+ * @param conf the Configuration object
+ * @return a unique temporary file
+ * @throws IOException IO problems
+ */
+ public static File createTmpFileForWrite(String path, long size,
+ Configuration conf) throws IOException {
+ if (conf.get(BUFFER_DIR_KEY) == null) {
+ conf.set(BUFFER_DIR_KEY, conf.get("hadoop.tmp.dir") + "/oss");
+ }
+ if (directoryAllocator == null) {
+ directoryAllocator = new LocalDirAllocator(BUFFER_DIR_KEY);
+ }
+ return directoryAllocator.createTmpFileForWrite(path, size, conf);
+ }
+
+ /**
+ * Get a integer option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static int intOption(Configuration conf, String key, int defVal, int min) {
+ int v = conf.getInt(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ LOG.debug("Value of {} is {}", key, v);
+ return v;
+ }
+
+ /**
+ * Get a long option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static long longOption(Configuration conf, String key, long defVal,
+ long min) {
+ long v = conf.getLong(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ LOG.debug("Value of {} is {}", key, v);
+ return v;
+ }
+
+ /**
+ * Get a size property from the configuration: this property must
+ * be at least equal to {@link Constants#MULTIPART_MIN_SIZE}.
+ * If it is too small, it is rounded up to that minimum, and a warning
+ * printed.
+ * @param conf configuration
+ * @param property property name
+ * @param defVal default value
+ * @return the value, guaranteed to be above the minimum size
+ */
+ public static long getMultipartSizeProperty(Configuration conf,
+ String property, long defVal) {
+ long partSize = conf.getLong(property, defVal);
+ if (partSize < MULTIPART_MIN_SIZE) {
+ LOG.warn("{} must be at least 100 KB; configured value is {}",
+ property, partSize);
+ partSize = MULTIPART_MIN_SIZE;
+ } else if (partSize > Integer.MAX_VALUE) {
+ LOG.warn("oss: {} capped to ~2.14GB(maximum allowed size with " +
+ "current output mechanism)", MULTIPART_UPLOAD_PART_SIZE_KEY);
+ partSize = Integer.MAX_VALUE;
+ }
+ return partSize;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
index 283927c..ecbd749 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
@@ -31,10 +31,10 @@ public final class Constants {
// User agent
public static final String USER_AGENT_PREFIX = "fs.oss.user.agent.prefix";
public static final String USER_AGENT_PREFIX_DEFAULT =
- VersionInfoUtils.getDefaultUserAgent();
+ VersionInfoUtils.getDefaultUserAgent();
// Class of credential provider
- public static final String ALIYUN_OSS_CREDENTIALS_PROVIDER_KEY =
+ public static final String CREDENTIALS_PROVIDER_KEY =
"fs.oss.credentials.provider";
// OSS access verification
@@ -82,10 +82,14 @@ public final class Constants {
public static final int MAX_PAGING_KEYS_DEFAULT = 1000;
// Size of each of or multipart pieces in bytes
- public static final String MULTIPART_UPLOAD_SIZE_KEY =
+ public static final String MULTIPART_UPLOAD_PART_SIZE_KEY =
"fs.oss.multipart.upload.size";
+ public static final long MULTIPART_UPLOAD_PART_SIZE_DEFAULT =
+ 104857600; // 100 MB
+
+ /** The minimum multipart size which Aliyun OSS supports. */
+ public static final int MULTIPART_MIN_SIZE = 100 * 1024;
- public static final long MULTIPART_UPLOAD_SIZE_DEFAULT = 10 * 1024 * 1024;
public static final int MULTIPART_UPLOAD_PART_NUM_LIMIT = 10000;
// Minimum size in bytes before we start a multipart uploads or copy
@@ -96,7 +100,6 @@ public final class Constants {
public static final String MULTIPART_DOWNLOAD_SIZE_KEY =
"fs.oss.multipart.download.size";
-
public static final long MULTIPART_DOWNLOAD_SIZE_DEFAULT = 512 * 1024;
public static final String MULTIPART_DOWNLOAD_THREAD_NUMBER_KEY =
@@ -139,9 +142,14 @@ public final class Constants {
public static final String FS_OSS_BLOCK_SIZE_KEY = "fs.oss.block.size";
public static final int FS_OSS_BLOCK_SIZE_DEFAULT = 64 * 1024 * 1024;
+
public static final String FS_OSS = "oss";
- public static final long MIN_MULTIPART_UPLOAD_PART_SIZE = 100 * 1024L;
- public static final int MAX_RETRIES = 10;
+ public static final String KEEPALIVE_TIME_KEY =
+ "fs.oss.threads.keepalivetime";
+ public static final int KEEPALIVE_TIME_DEFAULT = 60;
+ public static final String UPLOAD_ACTIVE_BLOCKS_KEY =
+ "fs.oss.upload.active.blocks";
+ public static final int UPLOAD_ACTIVE_BLOCKS_DEFAULT = 4;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
new file mode 100644
index 0000000..365d931
--- /dev/null
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSBlockOutputStream.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.aliyun.oss;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+
+import java.io.IOException;
+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.MULTIPART_UPLOAD_PART_SIZE_DEFAULT;
+
+/**
+ * Tests regular and multi-part upload functionality for
+ * AliyunOSSBlockOutputStream.
+ */
+public class TestAliyunOSSBlockOutputStream {
+ private FileSystem fs;
+ private static String testRootPath =
+ AliyunOSSTestUtils.generateUniqueTestPath();
+
+ @Rule
+ public Timeout testTimeout = new Timeout(30 * 60 * 1000);
+
+ @Before
+ public void setUp() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
+ conf.setInt(Constants.MULTIPART_UPLOAD_PART_SIZE_KEY, 5 * 1024 * 1024);
+ fs = AliyunOSSTestUtils.createTestFileSystem(conf);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (fs != null) {
+ fs.delete(new Path(testRootPath), true);
+ }
+ }
+
+ private Path getTestPath() {
+ return new Path(testRootPath + "/test-aliyun-oss");
+ }
+
+ @Test
+ public void testZeroByteUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 0);
+ }
+
+ @Test
+ public void testRegularUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024 - 1);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024 + 1);
+ }
+
+ @Test
+ public void testMultiPartUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
+ 6 * 1024 * 1024 - 1);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
+ 6 * 1024 * 1024 + 1);
+ }
+
+ @Test
+ public void testHugeUpload() throws IOException {
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
+ MULTIPART_UPLOAD_PART_SIZE_DEFAULT - 1);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
+ MULTIPART_UPLOAD_PART_SIZE_DEFAULT);
+ ContractTestUtils.createAndVerifyFile(fs, getTestPath(),
+ MULTIPART_UPLOAD_PART_SIZE_DEFAULT + 1);
+ }
+
+ @Test
+ public void testMultiPartUploadLimit() throws IOException {
+ long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024);
+ assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024);
+ assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024,
+ 100 * 1024);
+ assert(10000 * 100 * 1024 / partSize3
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+
+ long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024,
+ 100 * 1024);
+ assert(10001 * 100 * 1024 / partSize4
+ < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
index 66068c6..32d0e46 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSInputStream.java
@@ -123,15 +123,15 @@ public class TestAliyunOSSInputStream {
+ fsDataInputStream.getPos(), fsDataInputStream.getPos() == 0);
assertTrue("expected position at:"
- + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:"
- + in.getExpectNextPos(),
+ + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:"
+ + in.getExpectNextPos(),
in.getExpectNextPos() == Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT);
fsDataInputStream.seek(4 * 1024 * 1024);
assertTrue("expected position at:" + 4 * 1024 * 1024
- + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:"
- + in.getExpectNextPos(),
+ + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT + ", but got:"
+ + in.getExpectNextPos(),
in.getExpectNextPos() == 4 * 1024 * 1024
- + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT);
+ + Constants.MULTIPART_DOWNLOAD_SIZE_DEFAULT);
IOUtils.closeStream(fsDataInputStream);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
deleted file mode 100644
index 6b87d9c..0000000
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/TestAliyunOSSOutputStream.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.aliyun.oss;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.contract.ContractTestUtils;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Rule;
-import org.junit.Test;
-import org.junit.rules.Timeout;
-
-import java.io.IOException;
-
-/**
- * Tests regular and multi-part upload functionality for AliyunOSSOutputStream.
- */
-public class TestAliyunOSSOutputStream {
- private FileSystem fs;
- private static String testRootPath =
- AliyunOSSTestUtils.generateUniqueTestPath();
-
- @Rule
- public Timeout testTimeout = new Timeout(30 * 60 * 1000);
-
- @Before
- public void setUp() throws Exception {
- Configuration conf = new Configuration();
- conf.setLong(Constants.MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, 5 * 1024 * 1024);
- conf.setInt(Constants.MULTIPART_UPLOAD_SIZE_KEY, 5 * 1024 * 1024);
- fs = AliyunOSSTestUtils.createTestFileSystem(conf);
- }
-
- @After
- public void tearDown() throws Exception {
- if (fs != null) {
- fs.delete(new Path(testRootPath), true);
- }
- }
-
- protected Path getTestPath() {
- return new Path(testRootPath + "/test-aliyun-oss");
- }
-
- @Test
- public void testRegularUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUpload() throws IOException {
- ContractTestUtils.createAndVerifyFile(fs, getTestPath(), 6 * 1024 * 1024);
- }
-
- @Test
- public void testMultiPartUploadLimit() throws IOException {
- long partSize1 = AliyunOSSUtils.calculatePartSize(10 * 1024, 100 * 1024);
- assert(10 * 1024 / partSize1 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
-
- long partSize2 = AliyunOSSUtils.calculatePartSize(200 * 1024, 100 * 1024);
- assert(200 * 1024 / partSize2 < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
-
- long partSize3 = AliyunOSSUtils.calculatePartSize(10000 * 100 * 1024,
- 100 * 1024);
- assert(10000 * 100 * 1024 / partSize3
- < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
-
- long partSize4 = AliyunOSSUtils.calculatePartSize(10001 * 100 * 1024,
- 100 * 1024);
- assert(10001 * 100 * 1024 / partSize4
- < Constants.MULTIPART_UPLOAD_PART_NUM_LIMIT);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6542d17e/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
index 18d09d5..e9a98b3 100644
--- a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/contract/TestAliyunOSSContractDistCp.java
@@ -33,7 +33,7 @@ public class TestAliyunOSSContractDistCp extends AbstractContractDistCpTest {
protected Configuration createConfiguration() {
Configuration newConf = super.createConfiguration();
newConf.setLong(MIN_MULTIPART_UPLOAD_THRESHOLD_KEY, MULTIPART_SETTING);
- newConf.setLong(MULTIPART_UPLOAD_SIZE_KEY, MULTIPART_SETTING);
+ newConf.setLong(MULTIPART_UPLOAD_PART_SIZE_KEY, MULTIPART_SETTING);
return newConf;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-7840. Update PB for prefix
support of node attributes. Contributed by Naganarasimha G R.
Posted by na...@apache.org.
YARN-7840. Update PB for prefix support of node attributes. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d30e12d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d30e12d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d30e12d6
Branch: refs/heads/YARN-3409
Commit: d30e12d691ed694d9348a008b6db3a30f6630bb9
Parents: 9c77100
Author: bibinchundatt <bi...@apache.org>
Authored: Fri Feb 2 10:31:00 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 08:17:15 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/NodeAttribute.java | 22 ++++++++++-
.../src/main/proto/yarn_protos.proto | 7 ++--
.../records/impl/pb/NodeAttributePBImpl.java | 39 +++++++++++++++++---
.../hadoop/yarn/api/TestPBImplRecords.java | 7 ++--
4 files changed, 61 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 13081f3..01c70b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -37,15 +37,27 @@ import org.apache.hadoop.yarn.util.Records;
* Its not compulsory for all the attributes to have value, empty string is the
* default value of the <code>NodeAttributeType.STRING</code>
* </p>
- *
+ * <p>
+ * Node Attribute Prefix is used as namespace to segregate the attributes.
+ * </p>
*/
@Public
@Unstable
public abstract class NodeAttribute {
+ public static final String DEFAULT_PREFIX = "";
+
public static NodeAttribute newInstance(String attributeName,
NodeAttributeType attributeType, String attributeValue) {
+ return newInstance(DEFAULT_PREFIX, attributeName, attributeType,
+ attributeValue);
+ }
+
+ public static NodeAttribute newInstance(String attributePrefix,
+ String attributeName, NodeAttributeType attributeType,
+ String attributeValue) {
NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class);
+ nodeAttribute.setAttributePrefix(attributePrefix);
nodeAttribute.setAttributeName(attributeName);
nodeAttribute.setAttributeType(attributeType);
nodeAttribute.setAttributeValue(attributeValue);
@@ -54,6 +66,14 @@ public abstract class NodeAttribute {
@Public
@Unstable
+ public abstract String getAttributePrefix();
+
+ @Public
+ @Unstable
+ public abstract void setAttributePrefix(String attributePrefix);
+
+ @Public
+ @Unstable
public abstract String getAttributeName();
@Public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 46acffa..4bd90b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -376,9 +376,10 @@ enum NodeAttributeTypeProto {
}
message NodeAttributeProto {
- optional string attributeName = 1;
- optional NodeAttributeTypeProto attributeType = 2;
- optional string attributeValue = 3;
+ optional string attributePrefix = 1;
+ required string attributeName = 2;
+ optional NodeAttributeTypeProto attributeType = 3 [default = STRING];
+ optional string attributeValue = 4 [default=""];
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 11c9c48..7810939 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
@@ -63,6 +63,10 @@ public class NodeAttributePBImpl extends NodeAttribute {
@Override
public void setAttributeName(String attributeName) {
maybeInitBuilder();
+ if(attributeName == null) {
+ builder.clearAttributeName();
+ return;
+ }
builder.setAttributeName(attributeName);
}
@@ -78,6 +82,10 @@ public class NodeAttributePBImpl extends NodeAttribute {
@Override
public void setAttributeValue(String attributeValue) {
maybeInitBuilder();
+ if(attributeValue == null) {
+ builder.clearAttributeValue();
+ return;
+ }
builder.setAttributeValue(attributeValue);
}
@@ -111,12 +119,6 @@ public class NodeAttributePBImpl extends NodeAttribute {
}
@Override
- public String toString() {
- return " name-" + getAttributeName() + ":value-" + getAttributeValue()
- + ":type-" + getAttributeType();
- }
-
- @Override
public int hashCode() {
return getProto().hashCode();
}
@@ -152,4 +154,29 @@ public class NodeAttributePBImpl extends NodeAttribute {
return left.equals(right);
}
}
+
+ @Override
+ public String getAttributePrefix() {
+ NodeAttributeProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAttributePrefix()) {
+ return null;
+ }
+ return p.getAttributePrefix();
+ }
+
+ @Override
+ public void setAttributePrefix(String attributePrefix) {
+ maybeInitBuilder();
+ if(attributePrefix == null) {
+ builder.clearAttributePrefix();
+ return;
+ }
+ builder.setAttributePrefix(attributePrefix);
+ }
+
+ @Override
+ public String toString() {
+ return "Prefix-" + getAttributePrefix() + " :Name-" + getAttributeName()
+ + ":Value-" + getAttributeValue() + ":Type-" + getAttributeType();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d30e12d6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index c4438ad..f6f0d74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -1245,19 +1245,18 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
@Test
public void testNodeAttributePBImpl() throws Exception {
- validatePBImplRecord(NodeAttributePBImpl.class,
- NodeAttributeProto.class);
+ validatePBImplRecord(NodeAttributePBImpl.class, NodeAttributeProto.class);
}
@Test
public void testNodeToAttributesPBImpl() throws Exception {
validatePBImplRecord(NodeToAttributesPBImpl.class,
- NodeToAttributesProto.class);
+ NodeToAttributesProto.class);
}
@Test
public void testNodesToAttributesMappingRequestPBImpl() throws Exception {
validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class,
- NodesToAttributesMappingRequestProto.class);
+ NodesToAttributesMappingRequestProto.class);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-6856. [YARN-3409] Support CLI
for Node Attributes Mapping. Contributed by Naganarasimha G R.
Posted by na...@apache.org.
YARN-6856. [YARN-3409] Support CLI for Node Attributes Mapping. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6308b86a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6308b86a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6308b86a
Branch: refs/heads/YARN-3409
Commit: 6308b86a3bcfb1f6605113828332353602eea2f9
Parents: 461dcea
Author: Naganarasimha <na...@apache.org>
Authored: Tue Jan 23 07:18:20 2018 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 08:17:15 2018 +0800
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 2 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +
.../yarn/client/cli/NodeAttributesCLI.java | 410 +++++++++++++++++++
.../yarn/client/cli/TestNodeAttributesCLI.java | 328 +++++++++++++++
4 files changed, 744 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
index 9b7d7ba..8c92bd0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
@@ -575,7 +575,7 @@ public abstract class HAAdmin extends Configured implements Tool {
return 0;
}
- protected static class UsageInfo {
+ public static class UsageInfo {
public final String args;
public final String help;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 69afe6f..7cd838f 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -55,6 +55,7 @@ function hadoop_usage
hadoop_add_subcommand "timelinereader" client "run the timeline reader server"
hadoop_add_subcommand "timelineserver" daemon "run the timeline server"
hadoop_add_subcommand "top" client "view cluster information"
+ hadoop_add_subcommand "node-attributes" "map node to attibutes"
hadoop_add_subcommand "version" client "print the version"
hadoop_generate_usage "${HADOOP_SHELL_EXECNAME}" true
}
@@ -186,6 +187,10 @@ ${HADOOP_COMMON_HOME}/${HADOOP_COMMON_LIB_JARS_DIR}"
hadoop_add_classpath "$HADOOP_YARN_HOME/$YARN_DIR/timelineservice/lib/*"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
;;
+ node-attributes)
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false"
+ HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.NodeAttributesCLI'
+ ;;
timelineserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
new file mode 100644
index 0000000..2eff155
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -0,0 +1,410 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.MissingArgumentException;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.ha.HAAdmin.UsageInfo;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+
+/**
+ * CLI to map attributes to Nodes.
+ *
+ */
+public class NodeAttributesCLI extends Configured implements Tool {
+
+ protected static final String INVALID_MAPPING_ERR_MSG =
+ "Invalid Node to attribute mapping : ";
+
+ protected static final String USAGE_YARN_NODE_ATTRIBUTES =
+ "Usage: yarn node-attributes ";
+
+ protected static final String NO_MAPPING_ERR_MSG =
+ "No node-to-attributes mappings are specified";
+
+ protected final static Map<String, UsageInfo> NODE_ATTRIB_USAGE =
+ ImmutableMap.<String, UsageInfo>builder()
+ .put("-replace",
+ new UsageInfo(
+ "<\"node1:attribute[(type)][=value],attribute1[=value],"
+ + "attribute2 node2:attribute2[=value],attribute3\">",
+ " Replace the node to attributes mapping information at the"
+ + " ResourceManager with the new mapping. Currently"
+ + " supported attribute type. And string is the default"
+ + " type too. Attribute value if not specified for string"
+ + " type value will be considered as empty string."
+ + " Replaced node-attributes should not violate the"
+ + " existing attribute to attribute type mapping."))
+ .put("-add",
+ new UsageInfo(
+ "<\"node1:attribute[(type)][=value],attribute1[=value],"
+ + "attribute2 node2:attribute2[=value],attribute3\">",
+ " Adds or updates the node to attributes mapping information"
+ + " at the ResourceManager. Currently supported attribute"
+ + " type is string. And string is the default type too."
+ + " Attribute value if not specified for string type"
+ + " value will be considered as empty string. Added or"
+ + " updated node-attributes should not violate the"
+ + " existing attribute to attribute type mapping."))
+ .put("-remove",
+ new UsageInfo("<\"node1:attribute,attribute1 node2:attribute2\">",
+ " Removes the specified node to attributes mapping"
+ + " information at the ResourceManager"))
+ .put("-failOnUnknownNodes",
+ new UsageInfo("",
+ "Can be used optionally along with other options. When its"
+ + " set, it will fail if specified nodes are unknown."))
+ .build();
+
+ /** Output stream for errors, for use in tests. */
+ private PrintStream errOut = System.err;
+
+ public NodeAttributesCLI() {
+ super();
+ }
+
+ public NodeAttributesCLI(Configuration conf) {
+ super(conf);
+ }
+
+ protected void setErrOut(PrintStream errOut) {
+ this.errOut = errOut;
+ }
+
+ private void printHelpMsg(String cmd) {
+ StringBuilder builder = new StringBuilder();
+ UsageInfo usageInfo = null;
+ if (cmd != null && !(cmd.trim().isEmpty())) {
+ usageInfo = NODE_ATTRIB_USAGE.get(cmd);
+ }
+ if (usageInfo != null) {
+ if (usageInfo.args == null) {
+ builder.append(" " + cmd + ":\n" + usageInfo.help);
+ } else {
+ String space = (usageInfo.args == "") ? "" : " ";
+ builder.append(
+ " " + cmd + space + usageInfo.args + " :\n" + usageInfo.help);
+ }
+ } else {
+ // help for all commands
+ builder.append("Usage: yarn node-attributes\n");
+ for (Map.Entry<String, UsageInfo> cmdEntry : NODE_ATTRIB_USAGE
+ .entrySet()) {
+ usageInfo = cmdEntry.getValue();
+ builder.append(" " + cmdEntry.getKey() + " " + usageInfo.args
+ + " :\n " + usageInfo.help + "\n");
+ }
+ builder.append(" -help" + " [cmd]\n");
+ }
+ errOut.println(builder);
+ }
+
+ private static void buildIndividualUsageMsg(String cmd,
+ StringBuilder builder) {
+ UsageInfo usageInfo = NODE_ATTRIB_USAGE.get(cmd);
+ if (usageInfo == null) {
+ return;
+ }
+ if (usageInfo.args == null) {
+ builder.append(USAGE_YARN_NODE_ATTRIBUTES + cmd + "\n");
+ } else {
+ String space = (usageInfo.args == "") ? "" : " ";
+ builder.append(
+ USAGE_YARN_NODE_ATTRIBUTES + cmd + space + usageInfo.args + "\n");
+ }
+ }
+
+ private static void buildUsageMsgForAllCmds(StringBuilder builder) {
+ builder.append("Usage: yarn node-attributes\n");
+ for (Map.Entry<String, UsageInfo> cmdEntry : NODE_ATTRIB_USAGE.entrySet()) {
+ UsageInfo usageInfo = cmdEntry.getValue();
+ builder.append(" " + cmdEntry.getKey() + " " + usageInfo.args + "\n");
+ }
+ builder.append(" -help" + " [cmd]\n");
+ }
+
+ /**
+ * Displays format of commands.
+ *
+ * @param cmd The command that is being executed.
+ */
+ private void printUsage(String cmd) {
+ StringBuilder usageBuilder = new StringBuilder();
+ if (NODE_ATTRIB_USAGE.containsKey(cmd)) {
+ buildIndividualUsageMsg(cmd, usageBuilder);
+ } else {
+ buildUsageMsgForAllCmds(usageBuilder);
+ }
+ errOut.println(usageBuilder);
+ }
+
+ private void printUsage() {
+ printUsage("");
+ }
+
+ protected ResourceManagerAdministrationProtocol createAdminProtocol()
+ throws IOException {
+ // Get the current configuration
+ final YarnConfiguration conf = new YarnConfiguration(getConf());
+ return ClientRMProxy.createRMProxy(conf,
+ ResourceManagerAdministrationProtocol.class);
+ }
+
+ @Override
+ public void setConf(Configuration conf) {
+ if (conf != null) {
+ conf = addSecurityConfiguration(conf);
+ }
+ super.setConf(conf);
+ }
+
+ /**
+ * Add the requisite security principal settings to the given Configuration,
+ * returning a copy.
+ *
+ * @param conf the original config
+ * @return a copy with the security settings added
+ */
+ private static Configuration addSecurityConfiguration(Configuration conf) {
+ // Make a copy so we don't mutate it. Also use an YarnConfiguration to
+ // force loading of yarn-site.xml.
+ conf = new YarnConfiguration(conf);
+ conf.set(CommonConfigurationKeys.HADOOP_SECURITY_SERVICE_USER_NAME_KEY,
+ conf.get(YarnConfiguration.RM_PRINCIPAL, ""));
+ return conf;
+ }
+
+ @Override
+ public int run(String[] args) throws Exception {
+ if (args.length < 1) {
+ printUsage();
+ return -1;
+ }
+
+ int exitCode = -1;
+ int i = 0;
+ String cmd = args[i++];
+
+ if ("-help".equals(cmd)) {
+ exitCode = 0;
+ if (args.length >= 2) {
+ printHelpMsg(args[i]);
+ } else {
+ printHelpMsg("");
+ }
+ return exitCode;
+ }
+
+ try {
+ if ("-replace".equals(cmd)) {
+ exitCode = handleNodeAttributeMapping(args,
+ AttributeMappingOperationType.REPLACE);
+ } else if ("-add".equals(cmd)) {
+ exitCode =
+ handleNodeAttributeMapping(args, AttributeMappingOperationType.ADD);
+ } else if ("-remove".equals(cmd)) {
+ exitCode = handleNodeAttributeMapping(args,
+ AttributeMappingOperationType.REMOVE);
+ } else {
+ exitCode = -1;
+ errOut.println(cmd.substring(1) + ": Unknown command");
+ printUsage();
+ }
+ } catch (IllegalArgumentException arge) {
+ exitCode = -1;
+ errOut.println(cmd.substring(1) + ": " + arge.getLocalizedMessage());
+ printUsage(cmd);
+ } catch (RemoteException e) {
+ //
+ // This is a error returned by hadoop server. Print
+ // out the first line of the error message, ignore the stack trace.
+ exitCode = -1;
+ try {
+ String[] content;
+ content = e.getLocalizedMessage().split("\n");
+ errOut.println(cmd.substring(1) + ": " + content[0]);
+ } catch (Exception ex) {
+ errOut.println(cmd.substring(1) + ": " + ex.getLocalizedMessage());
+ }
+ } catch (Exception e) {
+ exitCode = -1;
+ errOut.println(cmd.substring(1) + ": " + e.getLocalizedMessage());
+ }
+ return exitCode;
+ }
+
+ private int handleNodeAttributeMapping(String args[],
+ AttributeMappingOperationType operation)
+ throws IOException, YarnException, ParseException {
+ Options opts = new Options();
+ opts.addOption(operation.name().toLowerCase(), true,
+ operation.name().toLowerCase());
+ opts.addOption("failOnUnknownNodes", false, "Fail on unknown nodes.");
+ int exitCode = -1;
+ CommandLine cliParser = null;
+ try {
+ cliParser = new GnuParser().parse(opts, args);
+ } catch (MissingArgumentException ex) {
+ errOut.println(NO_MAPPING_ERR_MSG);
+ printUsage(args[0]);
+ return exitCode;
+ }
+ List<NodeToAttributes> buildNodeLabelsMapFromStr =
+ buildNodeLabelsMapFromStr(
+ cliParser.getOptionValue(operation.name().toLowerCase()),
+ operation != AttributeMappingOperationType.REPLACE, operation);
+ NodesToAttributesMappingRequest request = NodesToAttributesMappingRequest
+ .newInstance(operation, buildNodeLabelsMapFromStr,
+ cliParser.hasOption("failOnUnknownNodes"));
+ ResourceManagerAdministrationProtocol adminProtocol = createAdminProtocol();
+ adminProtocol.mapAttributesToNodes(request);
+ return 0;
+ }
+
+ /**
+ * args are expected to be of the format
+ * node1:java(string)=8,ssd(boolean)=false node2:ssd(boolean)=true
+ */
+ private List<NodeToAttributes> buildNodeLabelsMapFromStr(String args,
+ boolean validateForAttributes, AttributeMappingOperationType operation) {
+ List<NodeToAttributes> nodeToAttributesList = new ArrayList<>();
+ for (String nodeToAttributesStr : args.split("[ \n]")) {
+ // for each node to attribute mapping
+ nodeToAttributesStr = nodeToAttributesStr.trim();
+ if (nodeToAttributesStr.isEmpty()
+ || nodeToAttributesStr.startsWith("#")) {
+ continue;
+ }
+ if (nodeToAttributesStr.indexOf(":") == -1) {
+ throw new IllegalArgumentException(
+ INVALID_MAPPING_ERR_MSG + nodeToAttributesStr);
+ }
+ String[] nodeToAttributes = nodeToAttributesStr.split(":");
+ Preconditions.checkArgument(!nodeToAttributes[0].trim().isEmpty(),
+ "Node name cannot be empty");
+ String node = nodeToAttributes[0];
+ String[] attributeNameValueType = null;
+ List<NodeAttribute> attributesList = new ArrayList<>();
+ NodeAttributeType attributeType = NodeAttributeType.STRING;
+ String attributeValue;
+ String attributeName;
+ Set<String> attributeNamesMapped = new HashSet<>();
+
+ String attributesStr[];
+ if (nodeToAttributes.length == 2) {
+ // fetching multiple attributes for a node
+ attributesStr = nodeToAttributes[1].split(",");
+ for (String attributeStr : attributesStr) {
+ // get information about each attribute.
+ attributeNameValueType = attributeStr.split("="); // to find name
+ // value
+ Preconditions.checkArgument(
+ !(attributeNameValueType[0] == null
+ || attributeNameValueType[0].isEmpty()),
+ "Attribute name cannot be null or empty");
+ attributeValue = attributeNameValueType.length > 1
+ ? attributeNameValueType[1] : "";
+ int indexOfOpenBracket = attributeNameValueType[0].indexOf("(");
+ if (indexOfOpenBracket == -1) {
+ attributeName = attributeNameValueType[0];
+ } else if (indexOfOpenBracket == 0) {
+ throw new IllegalArgumentException("Attribute for node " + node
+ + " is not properly configured : " + attributeStr);
+ } else {
+ // attribute type has been explicitly configured
+ int indexOfCloseBracket = attributeNameValueType[0].indexOf(")");
+ if (indexOfCloseBracket == -1
+ || indexOfCloseBracket < indexOfOpenBracket) {
+ throw new IllegalArgumentException("Attribute for node " + node
+ + " is not properly Configured : " + attributeStr);
+ }
+ String attributeTypeStr;
+ attributeName =
+ attributeNameValueType[0].substring(0, indexOfOpenBracket);
+ attributeTypeStr = attributeNameValueType[0]
+ .substring(indexOfOpenBracket + 1, indexOfCloseBracket);
+ try {
+ attributeType = NodeAttributeType
+ .valueOf(attributeTypeStr.trim().toUpperCase());
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "Invalid Attribute type configuration : " + attributeTypeStr
+ + " in " + attributeStr);
+ }
+ }
+ if (attributeNamesMapped.contains(attributeName)) {
+ throw new IllegalArgumentException("Attribute " + attributeName
+ + " has been mapped more than once in : "
+ + nodeToAttributesStr);
+ }
+ // TODO when we support different type of attribute type we need to
+ // cross verify whether input attributes itself is not violating
+ // attribute Name to Type mapping.
+ attributesList.add(NodeAttribute.newInstance(attributeName.trim(),
+ attributeType, attributeValue.trim()));
+ }
+ }
+ if (validateForAttributes) {
+ Preconditions.checkArgument((attributesList.size() > 0),
+ "Attributes cannot be null or empty for Operation "
+ + operation.name() + " on the node " + node);
+ }
+ nodeToAttributesList
+ .add(NodeToAttributes.newInstance(node, attributesList));
+ }
+
+ if (nodeToAttributesList.isEmpty()) {
+ throw new IllegalArgumentException(NO_MAPPING_ERR_MSG);
+ }
+ return nodeToAttributesList;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int result = ToolRunner.run(new NodeAttributesCLI(), args);
+ System.exit(result);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6308b86a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
new file mode 100644
index 0000000..cc92a93
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
@@ -0,0 +1,328 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.cli;
+
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+
+/**
+ * Test class for TestNodeAttributesCLI.
+ */
+public class TestNodeAttributesCLI {
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestNodeAttributesCLI.class);
+ private ResourceManagerAdministrationProtocol admin;
+ private NodesToAttributesMappingRequest request;
+ private NodeAttributesCLI nodeAttributesCLI;
+ private ByteArrayOutputStream errOutBytes = new ByteArrayOutputStream();
+ private String errOutput;
+
+ @Before
+ public void configure() throws IOException, YarnException {
+ admin = mock(ResourceManagerAdministrationProtocol.class);
+
+ when(admin.mapAttributesToNodes(any(NodesToAttributesMappingRequest.class)))
+ .thenAnswer(new Answer<NodesToAttributesMappingResponse>() {
+ @Override
+ public NodesToAttributesMappingResponse answer(
+ InvocationOnMock invocation) throws Throwable {
+ request =
+ (NodesToAttributesMappingRequest) invocation.getArguments()[0];
+ return NodesToAttributesMappingResponse.newInstance();
+ }
+ });
+
+ nodeAttributesCLI = new NodeAttributesCLI(new Configuration()) {
+ @Override
+ protected ResourceManagerAdministrationProtocol createAdminProtocol()
+ throws IOException {
+ return admin;
+ }
+ };
+
+ nodeAttributesCLI.setErrOut(new PrintStream(errOutBytes));
+ }
+
+ @Test
+ public void testHelp() throws Exception {
+ String[] args = new String[] { "-help", "-replace" };
+ assertTrue("It should have succeeded help for replace", 0 == runTool(args));
+ assertOutputContains(
+ "-replace <\"node1:attribute[(type)][=value],attribute1"
+ + "[=value],attribute2 node2:attribute2[=value],attribute3\"> :");
+ assertOutputContains("Replace the node to attributes mapping information at"
+ + " the ResourceManager with the new mapping. Currently supported"
+ + " attribute type. And string is the default type too. Attribute value"
+ + " if not specified for string type value will be considered as empty"
+ + " string. Replaced node-attributes should not violate the existing"
+ + " attribute to attribute type mapping.");
+
+ args = new String[] { "-help", "-remove" };
+ assertTrue("It should have succeeded help for replace", 0 == runTool(args));
+ assertOutputContains(
+ "-remove <\"node1:attribute,attribute1" + " node2:attribute2\"> :");
+ assertOutputContains("Removes the specified node to attributes mapping"
+ + " information at the ResourceManager");
+
+ args = new String[] { "-help", "-add" };
+ assertTrue("It should have succeeded help for replace", 0 == runTool(args));
+ assertOutputContains("-add <\"node1:attribute[(type)][=value],"
+ + "attribute1[=value],attribute2 node2:attribute2[=value],attribute3\">"
+ + " :");
+ assertOutputContains("Adds or updates the node to attributes mapping"
+ + " information at the ResourceManager. Currently supported attribute"
+ + " type is string. And string is the default type too. Attribute value"
+ + " if not specified for string type value will be considered as empty"
+ + " string. Added or updated node-attributes should not violate the"
+ + " existing attribute to attribute type mapping.");
+
+ args = new String[] { "-help", "-failOnUnknownNodes" };
+ assertTrue("It should have succeeded help for replace", 0 == runTool(args));
+ assertOutputContains("-failOnUnknownNodes :");
+ assertOutputContains("Can be used optionally along with other options. When"
+ + " its set, it will fail if specified nodes are unknown.");
+ }
+
+ @Test
+ public void testReplace() throws Exception {
+ // --------------------------------
+ // failure scenarios
+ // --------------------------------
+ // parenthesis not match
+ String[] args = new String[] { "-replace", "x(" };
+ assertTrue("It should have failed as no node is specified",
+ 0 != runTool(args));
+ assertFailureMessageContains(NodeAttributesCLI.INVALID_MAPPING_ERR_MSG);
+
+ // parenthesis not match
+ args = new String[] { "-replace", "x:(=abc" };
+ assertTrue(
+ "It should have failed as no closing parenthesis is not specified",
+ 0 != runTool(args));
+ assertFailureMessageContains(
+ "Attribute for node x is not properly configured : (=abc");
+
+ args = new String[] { "-replace", "x:()=abc" };
+ assertTrue("It should have failed as no type specified inside parenthesis",
+ 0 != runTool(args));
+ assertFailureMessageContains(
+ "Attribute for node x is not properly configured : ()=abc");
+
+ args = new String[] { "-replace", ":x(string)" };
+ assertTrue("It should have failed as no node is specified",
+ 0 != runTool(args));
+ assertFailureMessageContains("Node name cannot be empty");
+
+ // Not expected key=value specifying inner parenthesis
+ args = new String[] { "-replace", "x:(key=value)" };
+ assertTrue(0 != runTool(args));
+ assertFailureMessageContains(
+ "Attribute for node x is not properly configured : (key=value)");
+
+ // Should fail as no attributes specified
+ args = new String[] { "-replace" };
+ assertTrue("Should fail as no attribute mappings specified",
+ 0 != runTool(args));
+ assertFailureMessageContains(NodeAttributesCLI.NO_MAPPING_ERR_MSG);
+
+ // no labels, should fail
+ args = new String[] { "-replace", "-failOnUnknownNodes",
+ "x:key(string)=value,key2=val2" };
+ assertTrue("Should fail as no attribute mappings specified for replace",
+ 0 != runTool(args));
+ assertFailureMessageContains(NodeAttributesCLI.NO_MAPPING_ERR_MSG);
+
+ // no labels, should fail
+ args = new String[] { "-replace", " " };
+ assertTrue(0 != runTool(args));
+ assertFailureMessageContains(NodeAttributesCLI.NO_MAPPING_ERR_MSG);
+
+ args = new String[] { "-replace", ", " };
+ assertTrue(0 != runTool(args));
+ assertFailureMessageContains(NodeAttributesCLI.INVALID_MAPPING_ERR_MSG);
+ // --------------------------------
+ // success scenarios
+ // --------------------------------
+ args = new String[] { "-replace",
+ "x:key(string)=value,key2=val2 y:key2=val23,key3 z:key4" };
+ assertTrue("Should not fail as attribute has been properly mapped",
+ 0 == runTool(args));
+ List<NodeToAttributes> nodeAttributesList = new ArrayList<>();
+ List<NodeAttribute> attributes = new ArrayList<>();
+ attributes.add(
+ NodeAttribute.newInstance("key", NodeAttributeType.STRING, "value"));
+ attributes.add(
+ NodeAttribute.newInstance("key2", NodeAttributeType.STRING, "val2"));
+ nodeAttributesList.add(NodeToAttributes.newInstance("x", attributes));
+
+ // for node y
+ attributes = new ArrayList<>();
+ attributes.add(
+ NodeAttribute.newInstance("key2", NodeAttributeType.STRING, "val23"));
+ attributes
+ .add(NodeAttribute.newInstance("key3", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("y", attributes));
+
+ // for node y
+ attributes = new ArrayList<>();
+ attributes.add(
+ NodeAttribute.newInstance("key2", NodeAttributeType.STRING, "val23"));
+ attributes
+ .add(NodeAttribute.newInstance("key3", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("y", attributes));
+
+ // for node z
+ attributes = new ArrayList<>();
+ attributes
+ .add(NodeAttribute.newInstance("key4", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("z", attributes));
+
+ NodesToAttributesMappingRequest expected =
+ NodesToAttributesMappingRequest.newInstance(
+ AttributeMappingOperationType.REPLACE, nodeAttributesList, false);
+ assertTrue(request.equals(expected));
+ }
+
+ @Test
+ public void testRemove() throws Exception {
+ // --------------------------------
+ // failure scenarios
+ // --------------------------------
+ // parenthesis not match
+ String[] args = new String[] { "-remove", "x:" };
+ assertTrue("It should have failed as no node is specified",
+ 0 != runTool(args));
+ assertFailureMessageContains(
+ "Attributes cannot be null or empty for Operation REMOVE on the node x");
+ // --------------------------------
+ // success scenarios
+ // --------------------------------
+ args =
+ new String[] { "-remove", "x:key2,key3 z:key4", "-failOnUnknownNodes" };
+ assertTrue("Should not fail as attribute has been properly mapped",
+ 0 == runTool(args));
+ List<NodeToAttributes> nodeAttributesList = new ArrayList<>();
+ List<NodeAttribute> attributes = new ArrayList<>();
+ attributes
+ .add(NodeAttribute.newInstance("key2", NodeAttributeType.STRING, ""));
+ attributes
+ .add(NodeAttribute.newInstance("key3", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("x", attributes));
+
+ // for node z
+ attributes = new ArrayList<>();
+ attributes
+ .add(NodeAttribute.newInstance("key4", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("z", attributes));
+
+ NodesToAttributesMappingRequest expected =
+ NodesToAttributesMappingRequest.newInstance(
+ AttributeMappingOperationType.REMOVE, nodeAttributesList, true);
+ assertTrue(request.equals(expected));
+ }
+
+ @Test
+ public void testAdd() throws Exception {
+ // --------------------------------
+ // failure scenarios
+ // --------------------------------
+ // parenthesis not match
+ String[] args = new String[] { "-add", "x:" };
+ assertTrue("It should have failed as no node is specified",
+ 0 != runTool(args));
+ assertFailureMessageContains(
+ "Attributes cannot be null or empty for Operation ADD on the node x");
+ // --------------------------------
+ // success scenarios
+ // --------------------------------
+ args = new String[] { "-add", "x:key2=123,key3=abc z:key4(string)",
+ "-failOnUnknownNodes" };
+ assertTrue("Should not fail as attribute has been properly mapped",
+ 0 == runTool(args));
+ List<NodeToAttributes> nodeAttributesList = new ArrayList<>();
+ List<NodeAttribute> attributes = new ArrayList<>();
+ attributes.add(
+ NodeAttribute.newInstance("key2", NodeAttributeType.STRING, "123"));
+ attributes.add(
+ NodeAttribute.newInstance("key3", NodeAttributeType.STRING, "abc"));
+ nodeAttributesList.add(NodeToAttributes.newInstance("x", attributes));
+
+ // for node z
+ attributes = new ArrayList<>();
+ attributes
+ .add(NodeAttribute.newInstance("key4", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("z", attributes));
+
+ NodesToAttributesMappingRequest expected =
+ NodesToAttributesMappingRequest.newInstance(
+ AttributeMappingOperationType.ADD, nodeAttributesList, true);
+ assertTrue(request.equals(expected));
+ }
+
+ private void assertFailureMessageContains(String... messages) {
+ assertOutputContains(messages);
+ assertOutputContains(NodeAttributesCLI.USAGE_YARN_NODE_ATTRIBUTES);
+ }
+
+ private void assertOutputContains(String... messages) {
+ for (String message : messages) {
+ if (!errOutput.contains(message)) {
+ fail("Expected output to contain '" + message
+ + "' but err_output was:\n" + errOutput);
+ }
+ }
+ }
+
+ private int runTool(String... args) throws Exception {
+ errOutBytes.reset();
+ LOG.info("Running: NodeAttributesCLI " + Joiner.on(" ").join(args));
+ int ret = nodeAttributesCLI.run(args);
+ errOutput = new String(errOutBytes.toByteArray(), Charsets.UTF_8);
+ LOG.info("Err_output:\n" + errOutput);
+ return ret;
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-15352. Fix default local
maven repository path in create-release script. Contributed by Elek, Marton.
Posted by na...@apache.org.
HADOOP-15352. Fix default local maven repository path in create-release script. Contributed by Elek, Marton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/65907472
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/65907472
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/65907472
Branch: refs/heads/YARN-3409
Commit: 659074728e3b0985ca38919fb82ec71608a90a69
Parents: 1606dad
Author: Akira Ajisaka <aa...@apache.org>
Authored: Fri Mar 30 23:25:14 2018 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Fri Mar 30 23:25:14 2018 +0900
----------------------------------------------------------------------
dev-support/bin/create-release | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/65907472/dev-support/bin/create-release
----------------------------------------------------------------------
diff --git a/dev-support/bin/create-release b/dev-support/bin/create-release
index 949a46f..d3d96d9 100755
--- a/dev-support/bin/create-release
+++ b/dev-support/bin/create-release
@@ -389,7 +389,7 @@ function option_parse
hadoop_error "ERROR: Cannot set --mvncache and --dockercache simultaneously."
exit 1
else
- MVNCACHE=${MVNCACHE:-"${HOME}/.m2"}
+ MVNCACHE=${MVNCACHE:-"${HOME}/.m2/repository"}
fi
if [[ "${ASFRELEASE}" = true ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-8018. Added support for
initiating yarn service upgrade. Contributed by Chandni Singh
Posted by na...@apache.org.
YARN-8018. Added support for initiating yarn service upgrade.
Contributed by Chandni Singh
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27d60a16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27d60a16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27d60a16
Branch: refs/heads/YARN-3409
Commit: 27d60a16342fd39973d43b61008f54a8815a6237
Parents: edb202e
Author: Eric Yang <ey...@apache.org>
Authored: Mon Mar 26 18:46:31 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Mon Mar 26 18:46:31 2018 -0400
----------------------------------------------------------------------
.../yarn/service/client/ApiServiceClient.java | 18 ++
.../hadoop/yarn/service/webapp/ApiServer.java | 24 ++
.../hadoop/yarn/service/ClientAMProtocol.java | 12 +
.../hadoop/yarn/service/ClientAMService.java | 24 ++
.../hadoop/yarn/service/ServiceEvent.java | 49 ++++
.../hadoop/yarn/service/ServiceEventType.java | 28 +++
.../hadoop/yarn/service/ServiceManager.java | 225 +++++++++++++++++++
.../hadoop/yarn/service/ServiceScheduler.java | 19 ++
.../yarn/service/UpgradeComponentsFinder.java | 162 +++++++++++++
.../service/api/records/ComponentState.java | 2 +-
.../yarn/service/api/records/ServiceState.java | 2 +-
.../yarn/service/client/ServiceClient.java | 153 ++++++++++---
.../yarn/service/component/Component.java | 14 ++
.../yarn/service/component/ComponentEvent.java | 12 +
.../service/component/ComponentEventType.java | 4 +-
.../yarn/service/component/ComponentState.java | 3 +-
.../yarn/service/conf/YarnServiceConstants.java | 2 +
.../pb/client/ClientAMProtocolPBClientImpl.java | 26 +++
.../service/ClientAMProtocolPBServiceImpl.java | 24 ++
.../yarn/service/utils/CoreFileSystem.java | 31 ++-
.../yarn/service/utils/ServiceApiUtil.java | 28 ++-
.../src/main/proto/ClientAMProtocol.proto | 19 +-
.../hadoop/yarn/service/ServiceTestUtils.java | 90 +++++++-
.../TestDefaultUpgradeComponentsFinder.java | 63 ++++++
.../hadoop/yarn/service/TestServiceManager.java | 156 +++++++++++++
.../yarn/service/TestYarnNativeServices.java | 76 ++++---
.../yarn/service/client/TestServiceClient.java | 125 +++++++++++
.../yarn/service/utils/TestCoreFileSystem.java | 46 ++++
.../hadoop/yarn/client/api/AppAdminClient.java | 16 ++
29 files changed, 1374 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
index 49702e3..e4a245d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/client/ApiServiceClient.java
@@ -469,4 +469,22 @@ public class ApiServiceClient extends AppAdminClient {
return output;
}
+ @Override
+ public int actionUpgrade(String appName,
+ String fileName) throws IOException, YarnException {
+ int result;
+ try {
+ Service service =
+ loadAppJsonFromLocalFS(fileName, appName, null, null);
+ service.setState(ServiceState.UPGRADING);
+ String buffer = jsonSerDeser.toJson(service);
+ ClientResponse response = getApiClient()
+ .post(ClientResponse.class, buffer);
+ result = processResponse(response);
+ } catch (Exception e) {
+ LOG.error("Failed to upgrade application: ", e);
+ result = EXIT_EXCEPTION_THROWN;
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
index e7979b8..59ee05d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services-api/src/main/java/org/apache/hadoop/yarn/service/webapp/ApiServer.java
@@ -375,6 +375,12 @@ public class ApiServer {
&& updateServiceData.getLifetime() > 0) {
return updateLifetime(appName, updateServiceData, ugi);
}
+
+ // If an UPGRADE is requested
+ if (updateServiceData.getState() != null &&
+ updateServiceData.getState() == ServiceState.UPGRADING) {
+ return upgradeService(updateServiceData, ugi);
+ }
} catch (UndeclaredThrowableException e) {
return formatResponse(Status.BAD_REQUEST,
e.getCause().getMessage());
@@ -475,6 +481,24 @@ public class ApiServer {
return formatResponse(Status.OK, status);
}
+ private Response upgradeService(Service service,
+ final UserGroupInformation ugi) throws IOException, InterruptedException {
+ ServiceStatus status = new ServiceStatus();
+ ugi.doAs((PrivilegedExceptionAction<Void>) () -> {
+ ServiceClient sc = getServiceClient();
+ sc.init(YARN_CONFIG);
+ sc.start();
+ sc.actionUpgrade(service);
+ sc.close();
+ return null;
+ });
+ LOG.info("Service {} version {} upgrade initialized");
+ status.setDiagnostics("Service " + service.getName() +
+ " version " + service.getVersion() + " saved.");
+ status.setState(ServiceState.ACCEPTED);
+ return formatResponse(Status.ACCEPTED, status);
+ }
+
/**
* Used by negative test case.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
index 516d23d..4422451 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMProtocol.java
@@ -23,8 +23,14 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceResponseProto;
+
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceResponseProto;
+
import java.io.IOException;
@@ -37,4 +43,10 @@ public interface ClientAMProtocol {
StopResponseProto stop(StopRequestProto requestProto)
throws IOException, YarnException;
+
+ UpgradeServiceResponseProto upgrade(UpgradeServiceRequestProto request)
+ throws IOException, YarnException;
+
+ RestartServiceResponseProto restart(RestartServiceRequestProto request)
+ throws IOException, YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
index fb73f15..08c36f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ClientAMService.java
@@ -33,8 +33,12 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceResponseProto;
import org.apache.hadoop.yarn.service.component.ComponentEvent;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.slf4j.Logger;
@@ -142,4 +146,24 @@ public class ClientAMService extends AbstractService
public InetSocketAddress getBindAddress() {
return bindAddress;
}
+
+ @Override
+ public UpgradeServiceResponseProto upgrade(
+ UpgradeServiceRequestProto request) throws IOException {
+ ServiceEvent event = new ServiceEvent(ServiceEventType.UPGRADE);
+ event.setVersion(request.getVersion());
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ LOG.info("Upgrading service to version {} by {}", request.getVersion(),
+ UserGroupInformation.getCurrentUser());
+ return UpgradeServiceResponseProto.newBuilder().build();
+ }
+
+ @Override
+ public RestartServiceResponseProto restart(RestartServiceRequestProto request)
+ throws IOException, YarnException {
+ ServiceEvent event = new ServiceEvent(ServiceEventType.START);
+ context.scheduler.getDispatcher().getEventHandler().handle(event);
+ LOG.info("Restart service by {}", UserGroupInformation.getCurrentUser());
+ return RestartServiceResponseProto.newBuilder().build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
new file mode 100644
index 0000000..9e7d442
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEvent.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+/**
+ * Events are handled by {@link ServiceManager} to manage the service
+ * state.
+ */
+public class ServiceEvent extends AbstractEvent<ServiceEventType> {
+
+ private final ServiceEventType type;
+ private String version;
+
+ public ServiceEvent(ServiceEventType serviceEventType) {
+ super(serviceEventType);
+ this.type = serviceEventType;
+ }
+
+ public ServiceEventType getType() {
+ return type;
+ }
+
+ public String getVersion() {
+ return version;
+ }
+
+ public ServiceEvent setVersion(String version) {
+ this.version = version;
+ return this;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
new file mode 100644
index 0000000..2162eb5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceEventType.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+/**
+ * Types of {@link ServiceEvent}.
+ */
+public enum ServiceEventType {
+ START,
+ UPGRADE,
+ STOP_UPGRADE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
new file mode 100644
index 0000000..a3fbe89
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceManager.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.component.ComponentEvent;
+import org.apache.hadoop.yarn.service.component.ComponentEventType;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
+import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
+import org.apache.hadoop.yarn.state.MultipleArcTransition;
+import org.apache.hadoop.yarn.state.StateMachine;
+import org.apache.hadoop.yarn.state.StateMachineFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.text.MessageFormat;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Manages the state of the service.
+ */
+public class ServiceManager implements EventHandler<ServiceEvent> {
+ private static final Logger LOG = LoggerFactory.getLogger(
+ ServiceManager.class);
+
+ private final Service serviceSpec;
+ private final ServiceContext context;
+ private final ServiceScheduler scheduler;
+ private final ReentrantReadWriteLock.ReadLock readLock;
+ private final ReentrantReadWriteLock.WriteLock writeLock;
+
+ private final StateMachine<State, ServiceEventType, ServiceEvent>
+ stateMachine;
+
+ private final AsyncDispatcher dispatcher;
+ private final SliderFileSystem fs;
+ private final UpgradeComponentsFinder componentsFinder;
+
+ private String upgradeVersion;
+
+ private static final StateMachineFactory<ServiceManager, State,
+ ServiceEventType, ServiceEvent> STATE_MACHINE_FACTORY =
+ new StateMachineFactory<ServiceManager, State,
+ ServiceEventType, ServiceEvent>(State.STABLE)
+
+ .addTransition(State.STABLE, EnumSet.of(State.STABLE,
+ State.UPGRADING), ServiceEventType.UPGRADE,
+ new StartUpgradeTransition())
+
+ .addTransition(State.UPGRADING, EnumSet.of(State.STABLE,
+ State.UPGRADING), ServiceEventType.START,
+ new StopUpgradeTransition())
+ .installTopology();
+
+ public ServiceManager(ServiceContext context) {
+ Preconditions.checkNotNull(context);
+ this.context = context;
+ serviceSpec = context.service;
+ scheduler = context.scheduler;
+ stateMachine = STATE_MACHINE_FACTORY.make(this);
+ dispatcher = scheduler.getDispatcher();
+
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ fs = context.fs;
+ componentsFinder = new UpgradeComponentsFinder
+ .DefaultUpgradeComponentsFinder();
+ }
+
+ @Override
+ public void handle(ServiceEvent event) {
+ try {
+ writeLock.lock();
+ State oldState = getState();
+ try {
+ stateMachine.doTransition(event.getType(), event);
+ } catch (InvalidStateTransitionException e) {
+ LOG.error(MessageFormat.format(
+ "[SERVICE]: Invalid event {0} at {1}.", event.getType(),
+ oldState), e);
+ }
+ if (oldState != getState()) {
+ LOG.info("[SERVICE] Transitioned from {} to {} on {} event.",
+ oldState, getState(), event.getType());
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private State getState() {
+ this.readLock.lock();
+ try {
+ return this.stateMachine.getCurrentState();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ private static class StartUpgradeTransition implements
+ MultipleArcTransition<ServiceManager, ServiceEvent, State> {
+
+ @Override
+ public State transition(ServiceManager serviceManager,
+ ServiceEvent event) {
+ try {
+ Service targetSpec = ServiceApiUtil.loadServiceUpgrade(
+ serviceManager.fs, serviceManager.getName(), event.getVersion());
+
+ serviceManager.serviceSpec.setState(ServiceState.UPGRADING);
+ List<org.apache.hadoop.yarn.service.api.records.Component>
+ compsThatNeedUpgrade = serviceManager.componentsFinder.
+ findTargetComponentSpecs(serviceManager.serviceSpec, targetSpec);
+
+ if (compsThatNeedUpgrade != null && !compsThatNeedUpgrade.isEmpty()) {
+ compsThatNeedUpgrade.forEach(component -> {
+ ComponentEvent needUpgradeEvent = new ComponentEvent(
+ component.getName(), ComponentEventType.UPGRADE).
+ setTargetSpec(component);
+ serviceManager.dispatcher.getEventHandler().handle(
+ needUpgradeEvent);
+ });
+ }
+ serviceManager.upgradeVersion = event.getVersion();
+ return State.UPGRADING;
+ } catch (Throwable e) {
+ LOG.error("[SERVICE]: Upgrade to version {} failed", event.getVersion(),
+ e);
+ return State.STABLE;
+ }
+ }
+ }
+
+ private static class StopUpgradeTransition implements
+ MultipleArcTransition<ServiceManager, ServiceEvent, State> {
+
+ @Override
+ public State transition(ServiceManager serviceManager,
+ ServiceEvent event) {
+ //abort is not supported currently
+ //trigger re-check of service state
+ ServiceMaster.checkAndUpdateServiceState(serviceManager.scheduler,
+ true);
+ if (serviceManager.serviceSpec.getState().equals(ServiceState.STABLE)) {
+ return serviceManager.finalizeUpgrade() ? State.STABLE :
+ State.UPGRADING;
+ } else {
+ return State.UPGRADING;
+ }
+ }
+ }
+
+ /**
+ * @return whether finalization of upgrade was successful.
+ */
+ private boolean finalizeUpgrade() {
+ try {
+ Service upgradeSpec = ServiceApiUtil.loadServiceUpgrade(
+ fs, getName(), upgradeVersion);
+ ServiceApiUtil.writeAppDefinition(fs,
+ ServiceApiUtil.getServiceJsonPath(fs, getName()), upgradeSpec);
+ } catch (IOException e) {
+ LOG.error("Upgrade did not complete because unable to overwrite the" +
+ " service definition", e);
+ return false;
+ }
+
+ try {
+ fs.deleteClusterUpgradeDir(getName(), upgradeVersion);
+ } catch (IOException e) {
+ LOG.warn("Unable to delete upgrade definition for service {} " +
+ "version {}", getName(), upgradeVersion);
+ }
+ serviceSpec.setVersion(upgradeVersion);
+ upgradeVersion = null;
+ return true;
+ }
+
+ /**
+ * Returns the name of the service.
+ */
+ public String getName() {
+ return serviceSpec.getName();
+ }
+
+ /**
+ * State of {@link ServiceManager}.
+ */
+ public enum State {
+ STABLE, UPGRADING
+ }
+
+
+ @VisibleForTesting
+ Service getServiceSpec() {
+ return serviceSpec;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
index 6333197..79eef49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/ServiceScheduler.java
@@ -110,6 +110,9 @@ public class ServiceScheduler extends CompositeService {
LoggerFactory.getLogger(ServiceScheduler.class);
private Service app;
+ // This encapsulates the <code>app</code> with methods to upgrade the app.
+ private ServiceManager serviceManager;
+
// component_name -> component
private final Map<String, Component> componentsByName =
new ConcurrentHashMap<>();
@@ -192,6 +195,7 @@ public class ServiceScheduler extends CompositeService {
addIfService(nmClient);
dispatcher = new AsyncDispatcher("Component dispatcher");
+ dispatcher.register(ServiceEventType.class, new ServiceEventHandler());
dispatcher.register(ComponentEventType.class,
new ComponentEventHandler());
dispatcher.register(ComponentInstanceEventType.class,
@@ -300,6 +304,7 @@ public class ServiceScheduler extends CompositeService {
// Since AM has been started and registered, the service is in STARTED state
app.setState(ServiceState.STARTED);
+ serviceManager = new ServiceManager(context);
// recover components based on containers sent from RM
recoverComponents(response);
@@ -510,6 +515,20 @@ public class ServiceScheduler extends CompositeService {
}
}
+ private final class ServiceEventHandler
+ implements EventHandler<ServiceEvent> {
+ @Override
+ public void handle(ServiceEvent event) {
+ try {
+ serviceManager.handle(event);
+ } catch (Throwable t) {
+ LOG.error(MessageFormat
+ .format("[SERVICE]: Error in handling event type {0}",
+ event.getType()), t);
+ }
+ }
+ }
+
private final class ComponentEventHandler
implements EventHandler<ComponentEvent> {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
new file mode 100644
index 0000000..e18697b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/UpgradeComponentsFinder.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Service;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Finds all the the target component specs.
+ */
+public interface UpgradeComponentsFinder {
+
+ List<Component> findTargetComponentSpecs(Service currentDef,
+ Service targetDef);
+
+ /**
+ * Default implementation of {@link UpgradeComponentsFinder} that finds all
+ * the target component specs.
+ */
+ class DefaultUpgradeComponentsFinder implements UpgradeComponentsFinder {
+
+ @Override
+ public List<Component> findTargetComponentSpecs(Service currentDef,
+ Service targetDef) {
+ if (currentDef.getComponents().size() !=
+ targetDef.getComponents().size()) {
+ throw new UnsupportedOperationException(
+ "addition/deletion of components not supported by upgrade");
+ }
+ if (!currentDef.getKerberosPrincipal().equals(
+ targetDef.getKerberosPrincipal())) {
+ throw new UnsupportedOperationException("changes to kerberos " +
+ "principal not supported by upgrade");
+ }
+ if (!Objects.equals(currentDef.getQueue(), targetDef.getQueue())) {
+ throw new UnsupportedOperationException("changes to queue " +
+ "not supported by upgrade");
+ }
+ if (!Objects.equals(currentDef.getPlacementPolicy(),
+ targetDef.getPlacementPolicy())) {
+ throw new UnsupportedOperationException("changes to placement policy " +
+ "not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentDef.getResource(), targetDef.getResource())) {
+ throw new UnsupportedOperationException("changes to resource " +
+ "not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentDef.getDescription(),
+ targetDef.getDescription())) {
+ throw new UnsupportedOperationException("changes to description " +
+ "not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentDef.getQuicklinks(),
+ targetDef.getQuicklinks())) {
+ throw new UnsupportedOperationException("changes to quick links " +
+ "not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentDef.getLaunchTime(),
+ targetDef.getLaunchTime())) {
+ throw new UnsupportedOperationException("changes to launch time " +
+ "not supported by upgrade");
+ }
+
+
+ if (!Objects.equals(currentDef.getLifetime(),
+ targetDef.getLifetime())) {
+ throw new UnsupportedOperationException("changes to lifetime " +
+ "not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentDef.getConfiguration(),
+ currentDef.getConfiguration())) {
+ return targetDef.getComponents();
+ }
+
+ if (!Objects.equals(currentDef.getArtifact(), targetDef.getArtifact())) {
+ return targetDef.getComponents();
+ }
+
+ List<Component> targetComps = new ArrayList<>();
+ targetDef.getComponents().forEach(component -> {
+ Component currentComp = currentDef.getComponent(component.getName());
+
+ if (!Objects.equals(currentComp.getName(), component.getName())) {
+ throw new UnsupportedOperationException(
+ "changes to component name not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentComp.getDependencies(),
+ component.getDependencies())) {
+ throw new UnsupportedOperationException(
+ "changes to component dependencies not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentComp.getReadinessCheck(),
+ component.getReadinessCheck())) {
+ throw new UnsupportedOperationException(
+ "changes to component readiness check not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentComp.getResource(),
+ component.getResource())) {
+ throw new UnsupportedOperationException(
+ "changes to component resource not supported by upgrade");
+ }
+
+
+ if (!Objects.equals(currentComp.getRunPrivilegedContainer(),
+ component.getRunPrivilegedContainer())) {
+ throw new UnsupportedOperationException(
+ "changes to run privileged container not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentComp.getPlacementPolicy(),
+ component.getPlacementPolicy())) {
+ throw new UnsupportedOperationException(
+ "changes to component placement policy not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentComp.getQuicklinks(),
+ component.getQuicklinks())) {
+ throw new UnsupportedOperationException(
+ "changes to component quick links not supported by upgrade");
+ }
+
+ if (!Objects.equals(currentComp.getArtifact(),
+ component.getArtifact()) ||
+ !Objects.equals(currentComp.getLaunchCommand(),
+ component.getLaunchCommand()) ||
+ !Objects.equals(currentComp.getConfiguration(),
+ component.getConfiguration())) {
+ targetComps.add(component);
+ }
+ });
+ return targetComps;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
index 702a9ae..f7eda7b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ComponentState.java
@@ -26,5 +26,5 @@ import org.apache.hadoop.classification.InterfaceStability;
@InterfaceStability.Unstable
@ApiModel(description = "The current state of a component.")
public enum ComponentState {
- FLEXING, STABLE
+ FLEXING, STABLE, NEEDS_UPGRADE;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
index 902a0b1..286eaa2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/api/records/ServiceState.java
@@ -29,5 +29,5 @@ import org.apache.hadoop.classification.InterfaceStability;
@ApiModel(description = "The current state of an service.")
@javax.annotation.Generated(value = "class io.swagger.codegen.languages.JavaClientCodegen", date = "2016-06-02T08:15:05.615-07:00")
public enum ServiceState {
- ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX;
+ ACCEPTED, STARTED, STABLE, STOPPED, FAILED, FLEX, UPGRADING;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
index 5731e11..04ca943 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/client/ServiceClient.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.client;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
@@ -55,7 +56,9 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.ComponentCountProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceRequestProto;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import org.apache.hadoop.yarn.service.ServiceMaster;
import org.apache.hadoop.yarn.service.api.records.Component;
@@ -73,8 +76,8 @@ import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderUtils;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.ServiceRegistryUtils;
-import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.ServiceUtils;
+import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.service.utils.ZookeeperUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.Times;
@@ -186,6 +189,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return service;
}
+ @Override
public int actionSave(String fileName, String serviceName, Long lifetime,
String queue) throws IOException, YarnException {
return actionBuild(loadAppJsonFromLocalFS(fileName, serviceName,
@@ -194,9 +198,54 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
public int actionBuild(Service service)
throws YarnException, IOException {
- Path appDir = checkAppNotExistOnHdfs(service);
ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
- createDirAndPersistApp(appDir, service);
+ Path appDir = checkAppNotExistOnHdfs(service, false);
+ ServiceApiUtil.createDirAndPersistApp(fs, appDir, service);
+ return EXIT_SUCCESS;
+ }
+
+ @Override
+ public int actionUpgrade(String appName, String fileName)
+ throws IOException, YarnException {
+ checkAppExistOnHdfs(appName);
+ Service upgradeService = loadAppJsonFromLocalFS(fileName, appName,
+ null, null);
+ return actionUpgrade(upgradeService);
+ }
+
+ public int actionUpgrade(Service service) throws YarnException, IOException {
+ Service persistedService =
+ ServiceApiUtil.loadService(fs, service.getName());
+ if (!StringUtils.isEmpty(persistedService.getId())) {
+ cachedAppInfo.put(persistedService.getName(), new AppInfo(
+ ApplicationId.fromString(persistedService.getId()),
+ persistedService.getKerberosPrincipal().getPrincipalName()));
+ }
+
+ if (persistedService.getVersion().equals(service.getVersion())) {
+ String message =
+ service.getName() + " is already at version " + service.getVersion()
+ + ". There is nothing to upgrade.";
+ LOG.error(message);
+ throw new YarnException(message);
+ }
+
+ Path serviceUpgradeDir = checkAppNotExistOnHdfs(service, true);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ ServiceApiUtil.createDirAndPersistApp(fs, serviceUpgradeDir, service);
+
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(getAppId(service.getName()));
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ throw new YarnException(service.getName() + " AM hostname is empty");
+ }
+ ClientAMProtocol proxy = createAMProxy(service.getName(), appReport);
+
+ UpgradeServiceRequestProto.Builder requestBuilder =
+ UpgradeServiceRequestProto.newBuilder();
+ requestBuilder.setVersion(service.getVersion());
+
+ proxy.upgrade(requestBuilder.build());
return EXIT_SUCCESS;
}
@@ -212,16 +261,16 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
String serviceName = service.getName();
ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
verifyNoLiveAppInRM(serviceName, "create");
- Path appDir = checkAppNotExistOnHdfs(service);
+ Path appDir = checkAppNotExistOnHdfs(service, false);
// Write the definition first and then submit - AM will read the definition
- createDirAndPersistApp(appDir, service);
+ ServiceApiUtil.createDirAndPersistApp(fs, appDir, service);
ApplicationId appId = submitApp(service);
cachedAppInfo.put(serviceName, new AppInfo(appId, service
.getKerberosPrincipal().getPrincipalName()));
service.setId(appId.toString());
// update app definition with appId
- persistAppDef(appDir, service);
+ ServiceApiUtil.writeAppDefinition(fs, appDir, service);
return appId;
}
@@ -349,6 +398,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return original;
}
+ @Override
public int actionStop(String serviceName)
throws YarnException, IOException {
return actionStop(serviceName, true);
@@ -424,6 +474,7 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return EXIT_SUCCESS;
}
+ @Override
public int actionDestroy(String serviceName) throws YarnException,
IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
@@ -557,8 +608,8 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
}
}
- private ApplicationId submitApp(Service app)
- throws IOException, YarnException {
+ @VisibleForTesting
+ ApplicationId submitApp(Service app) throws IOException, YarnException {
String serviceName = app.getName();
Configuration conf = getConfig();
Path appRootDir = fs.buildClusterDirPath(app.getName());
@@ -772,29 +823,64 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return hasAMLog4j;
}
+ @Override
public int actionStart(String serviceName) throws YarnException, IOException {
ServiceApiUtil.validateNameFormat(serviceName, getConfig());
- Path appDir = checkAppExistOnHdfs(serviceName);
- Service service = ServiceApiUtil.loadService(fs, serviceName);
- ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
- // see if it is actually running and bail out;
- verifyNoLiveAppInRM(serviceName, "start");
- ApplicationId appId = submitApp(service);
- service.setId(appId.toString());
- // write app definition on to hdfs
- Path appJson = persistAppDef(appDir, service);
- LOG.info("Persisted service " + service.getName() + " at " + appJson);
- return 0;
+ Service liveService = getStatus(serviceName);
+ if (liveService == null ||
+ !liveService.getState().equals(ServiceState.UPGRADING)) {
+ Path appDir = checkAppExistOnHdfs(serviceName);
+ Service service = ServiceApiUtil.loadService(fs, serviceName);
+ ServiceApiUtil.validateAndResolveService(service, fs, getConfig());
+ // see if it is actually running and bail out;
+ verifyNoLiveAppInRM(serviceName, "start");
+ ApplicationId appId = submitApp(service);
+ service.setId(appId.toString());
+ // write app definition on to hdfs
+ Path appJson = ServiceApiUtil.writeAppDefinition(fs, appDir, service);
+ LOG.info("Persisted service " + service.getName() + " at " + appJson);
+ return 0;
+ } else {
+ LOG.info("Finalize service {} upgrade");
+ ApplicationReport appReport =
+ yarnClient.getApplicationReport(getAppId(serviceName));
+ if (StringUtils.isEmpty(appReport.getHost())) {
+ throw new YarnException(serviceName + " AM hostname is empty");
+ }
+ ClientAMProtocol proxy = createAMProxy(serviceName, appReport);
+
+ RestartServiceRequestProto.Builder requestBuilder =
+ RestartServiceRequestProto.newBuilder();
+ proxy.restart(requestBuilder.build());
+ return 0;
+ }
}
- private Path checkAppNotExistOnHdfs(Service service)
+ /**
+ * Verifies that the service definition does not exist on hdfs.
+ *
+ * @param service service
+ * @param isUpgrade true for upgrades; false otherwise
+ * @return path to the service definition..
+ * @throws IOException
+ * @throws SliderException
+ */
+ private Path checkAppNotExistOnHdfs(Service service, boolean isUpgrade)
throws IOException, SliderException {
- Path appDir = fs.buildClusterDirPath(service.getName());
+ Path appDir = !isUpgrade ? fs.buildClusterDirPath(service.getName()) :
+ fs.buildClusterUpgradeDirPath(service.getName(), service.getVersion());
fs.verifyDirectoryNonexistent(
new Path(appDir, service.getName() + ".json"));
return appDir;
}
+ /**
+ * Verifies that the service exists on hdfs.
+ * @param serviceName service name
+ * @return path to the service definition.
+ * @throws IOException
+ * @throws SliderException
+ */
private Path checkAppExistOnHdfs(String serviceName)
throws IOException, SliderException {
Path appDir = fs.buildClusterDirPath(serviceName);
@@ -802,20 +888,6 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
return appDir;
}
- private void createDirAndPersistApp(Path appDir, Service service)
- throws IOException, SliderException {
- FsPermission appDirPermission = new FsPermission("750");
- fs.createWithPermissions(appDir, appDirPermission);
- Path appJson = persistAppDef(appDir, service);
- LOG.info("Persisted service " + service.getName() + " at " + appJson);
- }
-
- private Path persistAppDef(Path appDir, Service service) throws IOException {
- Path appJson = new Path(appDir, service.getName() + ".json");
- jsonSerDeser.save(fs.getFileSystem(), appJson, service, true);
- return appJson;
- }
-
private void addHdfsDelegationTokenIfSecure(ContainerLaunchContext amContext)
throws IOException {
if (!UserGroupInformation.isSecurityEnabled()) {
@@ -1074,6 +1146,17 @@ public class ServiceClient extends AppAdminClient implements SliderExitCodes,
UserGroupInformation.getCurrentUser(), rpc, address);
}
+ @VisibleForTesting
+ void setFileSystem(SliderFileSystem fileSystem)
+ throws IOException {
+ this.fs = fileSystem;
+ }
+
+ @VisibleForTesting
+ void setYarnClient(YarnClient yarnClient) {
+ this.yarnClient = yarnClient;
+ }
+
public synchronized ApplicationId getAppId(String serviceName)
throws IOException, YarnException {
if (cachedAppInfo.containsKey(serviceName)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
index 3090692..0cd7e2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/Component.java
@@ -138,6 +138,12 @@ public class Component implements EventHandler<ComponentEvent> {
// For flex down, go to STABLE state
.addTransition(STABLE, EnumSet.of(STABLE, FLEXING),
FLEX, new FlexComponentTransition())
+ .addTransition(STABLE, UPGRADING, UPGRADE,
+ new ComponentNeedsUpgradeTransition())
+ .addTransition(FLEXING, UPGRADING, UPGRADE,
+ new ComponentNeedsUpgradeTransition())
+ .addTransition(UPGRADING, UPGRADING, UPGRADE,
+ new ComponentNeedsUpgradeTransition())
.installTopology();
public Component(
@@ -355,6 +361,14 @@ public class Component implements EventHandler<ComponentEvent> {
}
}
+ private static class ComponentNeedsUpgradeTransition extends BaseTransition {
+ @Override
+ public void transition(Component component, ComponentEvent event) {
+ component.componentSpec.setState(org.apache.hadoop.yarn.service.api.
+ records.ComponentState.NEEDS_UPGRADE);
+ }
+ }
+
public void removePendingInstance(ComponentInstance instance) {
pendingInstances.remove(instance);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
index 447b436..7bd5cb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEvent.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service.component;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -32,6 +33,7 @@ public class ComponentEvent extends AbstractEvent<ComponentEventType> {
private ComponentInstance instance;
private ContainerStatus status;
private ContainerId containerId;
+ private org.apache.hadoop.yarn.service.api.records.Component targetSpec;
public ContainerId getContainerId() {
return containerId;
@@ -91,4 +93,14 @@ public class ComponentEvent extends AbstractEvent<ComponentEventType> {
this.status = status;
return this;
}
+
+ public org.apache.hadoop.yarn.service.api.records.Component getTargetSpec() {
+ return targetSpec;
+ }
+
+ public ComponentEvent setTargetSpec(
+ org.apache.hadoop.yarn.service.api.records.Component targetSpec) {
+ this.targetSpec = Preconditions.checkNotNull(targetSpec);
+ return this;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
index 067302d..970788a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentEventType.java
@@ -23,5 +23,7 @@ public enum ComponentEventType {
CONTAINER_ALLOCATED,
CONTAINER_RECOVERED,
CONTAINER_STARTED,
- CONTAINER_COMPLETED
+ CONTAINER_COMPLETED,
+ UPGRADE,
+ STOP_UPGRADE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
index a5f9ff4..0f63d03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/component/ComponentState.java
@@ -21,5 +21,6 @@ package org.apache.hadoop.yarn.service.component;
public enum ComponentState {
INIT,
FLEXING,
- STABLE
+ STABLE,
+ UPGRADING
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
index 0378d24..7b474f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/conf/YarnServiceConstants.java
@@ -92,4 +92,6 @@ public interface YarnServiceConstants {
String CONTENT = "content";
String PRINCIPAL = "yarn.service.am.principal";
+
+ String UPGRADE_DIR = "upgrade";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
index 33e33a6..8152225 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/client/ClientAMProtocolPBClientImpl.java
@@ -35,8 +35,12 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
import org.apache.hadoop.yarn.service.impl.pb.service.ClientAMProtocolPB;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.StopRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceResponseProto;
public class ClientAMProtocolPBClientImpl
implements ClientAMProtocol, Closeable {
@@ -88,4 +92,26 @@ public class ClientAMProtocolPBClientImpl
RPC.stopProxy(this.proxy);
}
}
+
+ @Override
+ public UpgradeServiceResponseProto upgrade(
+ UpgradeServiceRequestProto request) throws IOException, YarnException {
+ try {
+ return proxy.upgradeService(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
+
+ @Override
+ public RestartServiceResponseProto restart(RestartServiceRequestProto request)
+ throws IOException, YarnException {
+ try {
+ return proxy.restartService(null, request);
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
index 7100781..1a1a1ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/impl/pb/service/ClientAMProtocolPBServiceImpl.java
@@ -25,6 +25,10 @@ import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.FlexComponentsResponseProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusRequestProto;
import org.apache.hadoop.yarn.proto.ClientAMProtocol.GetStatusResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.RestartServiceResponseProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceRequestProto;
+import org.apache.hadoop.yarn.proto.ClientAMProtocol.UpgradeServiceResponseProto;
import org.apache.hadoop.yarn.service.ClientAMProtocol;
import java.io.IOException;
@@ -67,4 +71,24 @@ public class ClientAMProtocolPBServiceImpl implements ClientAMProtocolPB {
throw new ServiceException(e);
}
}
+
+ @Override
+ public UpgradeServiceResponseProto upgradeService(RpcController controller,
+ UpgradeServiceRequestProto request) throws ServiceException {
+ try {
+ return real.upgrade(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
+ public RestartServiceResponseProto restartService(RpcController controller,
+ RestartServiceRequestProto request) throws ServiceException {
+ try {
+ return real.restart(request);
+ } catch (IOException | YarnException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
index 284825e..5c2bac6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/CoreFileSystem.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.IOUtils;
-import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
@@ -112,11 +111,39 @@ public class CoreFileSystem {
public Path buildClusterDirPath(String clustername) {
Preconditions.checkNotNull(clustername);
Path path = getBaseApplicationPath();
- return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/" + clustername);
+ return new Path(path, YarnServiceConstants.SERVICES_DIRECTORY + "/"
+ + clustername);
}
+ /**
+ * Build up the upgrade path string for a cluster. No attempt to
+ * create the directory is made.
+ *
+ * @param clusterName name of the cluster
+ * @param version version of the cluster
+ * @return the upgrade path to the cluster
+ */
+ public Path buildClusterUpgradeDirPath(String clusterName, String version) {
+ Preconditions.checkNotNull(clusterName);
+ Preconditions.checkNotNull(version);
+ return new Path(buildClusterDirPath(clusterName),
+ YarnServiceConstants.UPGRADE_DIR + "/" + version);
+ }
/**
+ * Delete the upgrade cluster directory.
+ * @param clusterName name of the cluster
+ * @param version version of the cluster
+ * @throws IOException
+ */
+ public void deleteClusterUpgradeDir(String clusterName, String version)
+ throws IOException {
+ Preconditions.checkNotNull(clusterName);
+ Preconditions.checkNotNull(version);
+ Path upgradeCluster = buildClusterUpgradeDirPath(clusterName, version);
+ fileSystem.delete(upgradeCluster, true);
+ }
+ /**
* Build up the path string for keytab install location -no attempt to
* create the directory is made
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
index 0591775..13d9a37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/java/org/apache/hadoop/yarn/service/utils/ServiceApiUtil.java
@@ -22,9 +22,9 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.registry.client.api.RegistryConstants;
import org.apache.hadoop.registry.client.binding.RegistryUtils;
-import org.apache.hadoop.security.HadoopKerberosName;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.service.api.records.Service;
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.service.api.records.Artifact;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Configuration;
import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.provider.AbstractClientProvider;
import org.apache.hadoop.yarn.service.provider.ProviderFactory;
import org.apache.hadoop.yarn.service.monitor.probe.MonitorUtils;
@@ -275,6 +276,14 @@ public class ServiceApiUtil {
return jsonSerDeser.load(fs.getFileSystem(), serviceJson);
}
+ public static Service loadServiceUpgrade(SliderFileSystem fs,
+ String serviceName, String version) throws IOException {
+ Path versionPath = fs.buildClusterUpgradeDirPath(serviceName, version);
+ Path versionedDef = new Path(versionPath, serviceName + ".json");
+ LOG.info("Loading service definition from {}", versionedDef);
+ return jsonSerDeser.load(fs.getFileSystem(), versionedDef);
+ }
+
public static Service loadServiceFrom(SliderFileSystem fs,
Path appDefPath) throws IOException {
LOG.info("Loading service definition from " + appDefPath);
@@ -429,6 +438,23 @@ public class ServiceApiUtil {
return sortByDependencies(components, sortedComponents);
}
+ public static void createDirAndPersistApp(SliderFileSystem fs, Path appDir,
+ Service service)
+ throws IOException, SliderException {
+ FsPermission appDirPermission = new FsPermission("750");
+ fs.createWithPermissions(appDir, appDirPermission);
+ Path appJson = writeAppDefinition(fs, appDir, service);
+ LOG.info("Persisted service {} version {} at {}", service.getName(),
+ service.getVersion(), appJson);
+ }
+
+ public static Path writeAppDefinition(SliderFileSystem fs, Path appDir,
+ Service service) throws IOException {
+ Path appJson = new Path(appDir, service.getName() + ".json");
+ jsonSerDeser.save(fs.getFileSystem(), appJson, service, true);
+ return appJson;
+ }
+
public static String $(String s) {
return "${" + s +"}";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
index 0a21c24..3677593 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/main/proto/ClientAMProtocol.proto
@@ -26,6 +26,10 @@ service ClientAMProtocolService {
rpc flexComponents(FlexComponentsRequestProto) returns (FlexComponentsResponseProto);
rpc getStatus(GetStatusRequestProto) returns (GetStatusResponseProto);
rpc stop(StopRequestProto) returns (StopResponseProto);
+ rpc upgradeService(UpgradeServiceRequestProto)
+ returns (UpgradeServiceResponseProto);
+ rpc restartService(RestartServiceRequestProto)
+ returns (RestartServiceResponseProto);
}
message FlexComponentsRequestProto {
@@ -37,7 +41,7 @@ message ComponentCountProto {
optional int64 numberOfContainers = 2;
}
-message FlexComponentsResponseProto{
+message FlexComponentsResponseProto {
}
message GetStatusRequestProto {
@@ -53,4 +57,17 @@ message StopRequestProto {
message StopResponseProto {
+}
+
+message UpgradeServiceRequestProto {
+ optional string version = 1;
+}
+
+message UpgradeServiceResponseProto {
+}
+
+message RestartServiceRequestProto {
+}
+
+message RestartServiceResponseProto {
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
index 9933211..8347eb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/ServiceTestUtils.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.service;
+import com.google.common.base.Throwables;
import org.apache.commons.io.FileUtils;
import org.apache.curator.test.TestingCluster;
import org.apache.hadoop.conf.Configuration;
@@ -26,18 +27,23 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.yarn.service.api.records.Service;
-import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Resource;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.client.ServiceClient;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConf;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
import org.apache.hadoop.yarn.service.utils.JsonSerDeser;
import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.codehaus.jackson.map.PropertyNamingStrategy;
+import org.junit.rules.TestWatcher;
+import org.junit.runner.Description;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -47,6 +53,8 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.URL;
+import java.nio.file.Paths;
+import java.util.Map;
import static org.apache.hadoop.registry.client.api.RegistryConstants.KEY_REGISTRY_ZK_QUORUM;
import static org.apache.hadoop.yarn.conf.YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC;
@@ -78,7 +86,7 @@ public class ServiceTestUtils {
// Example service definition
// 2 components, each of which has 2 containers.
- protected Service createExampleApplication() {
+ public static Service createExampleApplication() {
Service exampleApp = new Service();
exampleApp.setName("example-app");
exampleApp.setVersion("v1");
@@ -176,7 +184,7 @@ public class ServiceTestUtils {
zkCluster.start();
conf.set(YarnConfiguration.RM_ZK_ADDRESS, zkCluster.getConnectString());
conf.set(KEY_REGISTRY_ZK_QUORUM, zkCluster.getConnectString());
- LOG.info("ZK cluster: " + zkCluster.getConnectString());
+ LOG.info("ZK cluster: " + zkCluster.getConnectString());
fs = FileSystem.get(conf);
basedir = new File("target", "apps");
@@ -268,4 +276,78 @@ public class ServiceTestUtils {
}
}
+ /**
+ * Creates a {@link ServiceClient} for test purposes.
+ */
+ public static ServiceClient createClient(Configuration conf)
+ throws Exception {
+ ServiceClient client = new ServiceClient() {
+ @Override
+ protected Path addJarResource(String appName,
+ Map<String, LocalResource> localResources)
+ throws IOException, SliderException {
+ // do nothing, the Unit test will use local jars
+ return null;
+ }
+ };
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
+
+ /**
+ * Watcher to initialize yarn service base path under target and deletes the
+ * the test directory when finishes.
+ */
+ public static class ServiceFSWatcher extends TestWatcher {
+ private YarnConfiguration conf;
+ private SliderFileSystem fs;
+ private java.nio.file.Path serviceBasePath;
+
+ @Override
+ protected void starting(Description description) {
+ conf = new YarnConfiguration();
+ delete(description);
+ serviceBasePath = Paths.get("target",
+ description.getClassName(), description.getMethodName());
+ conf.set(YARN_SERVICE_BASE_PATH, serviceBasePath.toString());
+ try {
+ fs = new SliderFileSystem(conf);
+ } catch (IOException e) {
+ Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ protected void finished(Description description) {
+ delete(description);
+ }
+
+ private void delete(Description description) {
+ FileUtils.deleteQuietly(Paths.get("target",
+ description.getClassName()).toFile());
+ }
+
+ /**
+ * Returns the yarn conf.
+ */
+ public YarnConfiguration getConf() {
+ return conf;
+ }
+
+ /**
+ * Returns the file system.
+ */
+ public SliderFileSystem getFs() {
+ return fs;
+ }
+
+ /**
+ * Returns the test service base path.
+ */
+ public java.nio.file.Path getServiceBasePath() {
+ return serviceBasePath;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
new file mode 100644
index 0000000..086fdbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestDefaultUpgradeComponentsFinder.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Tests for {@link UpgradeComponentsFinder.DefaultUpgradeComponentsFinder}.
+ */
+public class TestDefaultUpgradeComponentsFinder {
+
+ private UpgradeComponentsFinder.DefaultUpgradeComponentsFinder finder =
+ new UpgradeComponentsFinder.DefaultUpgradeComponentsFinder();
+
+ @Test
+ public void testServiceArtifactChange() {
+ Service currentDef = ServiceTestUtils.createExampleApplication();
+ Service targetDef = ServiceTestUtils.createExampleApplication();
+ targetDef.getComponents().forEach(x -> x.setArtifact(
+ TestServiceManager.createTestArtifact("v1")));
+
+ Assert.assertEquals("all components need upgrade",
+ targetDef.getComponents(), finder.findTargetComponentSpecs(currentDef,
+ targetDef));
+ }
+
+ @Test
+ public void testComponentArtifactChange() {
+ Service currentDef = TestServiceManager.createBaseDef("test");
+ Service targetDef = TestServiceManager.createBaseDef("test");
+
+ targetDef.getComponents().get(0).setArtifact(
+ TestServiceManager.createTestArtifact("v2"));
+
+ List<Component> expected = new ArrayList<>();
+ expected.add(targetDef.getComponents().get(0));
+
+ Assert.assertEquals("single components needs upgrade",
+ expected, finder.findTargetComponentSpecs(currentDef,
+ targetDef));
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
new file mode 100644
index 0000000..c65a5d4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestServiceManager.java
@@ -0,0 +1,156 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.registry.client.api.RegistryOperations;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.service.api.records.Artifact;
+import org.apache.hadoop.yarn.service.api.records.ComponentState;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.api.records.ServiceState;
+import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.registry.YarnRegistryViewForProviders;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.util.Map;
+
+import static org.mockito.Mockito.mock;
+
+/**
+ * Tests for {@link ServiceManager}.
+ */
+public class TestServiceManager {
+
+ @Rule
+ public ServiceTestUtils.ServiceFSWatcher rule =
+ new ServiceTestUtils.ServiceFSWatcher();
+
+ @Test
+ public void testUpgrade() throws IOException, SliderException {
+ ServiceManager serviceManager = createTestServiceManager("testUpgrade");
+ upgrade(serviceManager, "v2", false);
+ Assert.assertEquals("service not upgraded", ServiceState.UPGRADING,
+ serviceManager.getServiceSpec().getState());
+ }
+
+ @Test
+ public void testRestartNothingToUpgrade()
+ throws IOException, SliderException {
+ ServiceManager serviceManager = createTestServiceManager("testRestart");
+ upgrade(serviceManager, "v2", false);
+
+ //make components stable
+ serviceManager.getServiceSpec().getComponents().forEach(comp -> {
+ comp.setState(ComponentState.STABLE);
+ });
+ serviceManager.handle(new ServiceEvent(ServiceEventType.START));
+ Assert.assertEquals("service not re-started", ServiceState.STABLE,
+ serviceManager.getServiceSpec().getState());
+ }
+
+ @Test
+ public void testRestartWithPendingUpgrade()
+ throws IOException, SliderException {
+ ServiceManager serviceManager = createTestServiceManager("testRestart");
+ upgrade(serviceManager, "v2", true);
+ serviceManager.handle(new ServiceEvent(ServiceEventType.START));
+ Assert.assertEquals("service should still be upgrading",
+ ServiceState.UPGRADING, serviceManager.getServiceSpec().getState());
+ }
+
+
+ private void upgrade(ServiceManager service, String version,
+ boolean upgradeArtifact)
+ throws IOException, SliderException {
+ Service upgradedDef = ServiceTestUtils.createExampleApplication();
+ upgradedDef.setName(service.getName());
+ upgradedDef.setVersion(version);
+ if (upgradeArtifact) {
+ Artifact upgradedArtifact = createTestArtifact("2");
+ upgradedDef.getComponents().forEach(component -> {
+ component.setArtifact(upgradedArtifact);
+ });
+ }
+ writeUpgradedDef(upgradedDef);
+ ServiceEvent upgradeEvent = new ServiceEvent(ServiceEventType.UPGRADE);
+ upgradeEvent.setVersion("v2");
+ service.handle(upgradeEvent);
+ }
+
+ private ServiceManager createTestServiceManager(String name)
+ throws IOException {
+ ServiceContext context = new ServiceContext();
+ context.service = createBaseDef(name);
+ context.fs = rule.getFs();
+
+ context.scheduler = new ServiceScheduler(context) {
+ @Override
+ protected YarnRegistryViewForProviders createYarnRegistryOperations(
+ ServiceContext context, RegistryOperations registryClient) {
+ return mock(YarnRegistryViewForProviders.class);
+ }
+ };
+
+ context.scheduler.init(rule.getConf());
+
+ Map<String, org.apache.hadoop.yarn.service.component.Component>
+ componentState = context.scheduler.getAllComponents();
+ context.service.getComponents().forEach(component -> {
+ componentState.put(component.getName(),
+ new org.apache.hadoop.yarn.service.component.Component(component,
+ 1L, context));
+ });
+ return new ServiceManager(context);
+ }
+
+ static Service createBaseDef(String name) {
+ ApplicationId applicationId = ApplicationId.newInstance(
+ System.currentTimeMillis(), 1);
+ Service serviceDef = ServiceTestUtils.createExampleApplication();
+ serviceDef.setId(applicationId.toString());
+ serviceDef.setName(name);
+ serviceDef.setState(ServiceState.STARTED);
+ Artifact artifact = createTestArtifact("1");
+
+ serviceDef.getComponents().forEach(component ->
+ component.setArtifact(artifact));
+ return serviceDef;
+ }
+
+ static Artifact createTestArtifact(String artifactId) {
+ Artifact artifact = new Artifact();
+ artifact.setId(artifactId);
+ artifact.setType(Artifact.TypeEnum.TARBALL);
+ return artifact;
+ }
+
+ private void writeUpgradedDef(Service upgradedDef)
+ throws IOException, SliderException {
+ Path upgradePath = rule.getFs().buildClusterUpgradeDirPath(
+ upgradedDef.getName(), upgradedDef.getVersion());
+ ServiceApiUtil.createDirAndPersistApp(rule.getFs(), upgradePath,
+ upgradedDef);
+ }
+
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-6855. [YARN-3409] CLI Proto
Modifications to support Node Attributes. Contributed by Naganarasimha G R.
Posted by na...@apache.org.
YARN-6855. [YARN-3409] CLI Proto Modifications to support Node Attributes. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/461dcea8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/461dcea8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/461dcea8
Branch: refs/heads/YARN-3409
Commit: 461dcea80b301307bb57a0e06ebb786a6476c3d1
Parents: dc8e343
Author: Naganarasimha <na...@apache.org>
Authored: Sun Jan 21 00:53:02 2018 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 08:17:15 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/NodeAttribute.java | 78 ++++++++
.../yarn/api/records/NodeAttributeType.java | 35 ++++
.../ResourceManagerAdministrationProtocol.java | 13 +-
.../AttributeMappingOperationType.java | 42 ++++
.../api/protocolrecords/NodeToAttributes.java | 59 ++++++
.../NodesToAttributesMappingRequest.java | 69 +++++++
.../NodesToAttributesMappingResponse.java | 27 +++
...esourcemanager_administration_protocol.proto | 1 +
..._server_resourcemanager_service_protos.proto | 21 ++
.../src/main/proto/yarn_protos.proto | 11 ++
.../records/impl/pb/NodeAttributePBImpl.java | 155 +++++++++++++++
...nagerAdministrationProtocolPBClientImpl.java | 26 ++-
...agerAdministrationProtocolPBServiceImpl.java | 31 ++-
.../impl/pb/NodeToAttributesPBImpl.java | 161 +++++++++++++++
.../NodesToAttributesMappingRequestPBImpl.java | 194 +++++++++++++++++++
.../NodesToAttributesMappingResponsePBImpl.java | 47 +++++
.../hadoop/yarn/api/TestPBImplRecords.java | 34 +++-
.../yarn/server/MockResourceManagerFacade.java | 17 +-
.../server/resourcemanager/AdminService.java | 10 +
.../DefaultRMAdminRequestInterceptor.java | 9 +
.../router/rmadmin/RouterRMAdminService.java | 10 +
.../PassThroughRMAdminRequestInterceptor.java | 9 +
22 files changed, 1044 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
new file mode 100644
index 0000000..13081f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * Node Attribute is a kind of a label which represents one of the
+ * attribute/feature of a Node. Its different from node partition label as
+ * resource guarantees across the queues will not be maintained for these type
+ * of labels.
+ * </p>
+ * <p>
+ * A given Node can be mapped with any kind of attribute, few examples are
+ * HAS_SSD=true, JAVA_VERSION=JDK1.8, OS_TYPE=WINDOWS.
+ * </p>
+ * <p>
+ * Its not compulsory for all the attributes to have value, empty string is the
+ * default value of the <code>NodeAttributeType.STRING</code>
+ * </p>
+ *
+ */
+@Public
+@Unstable
+public abstract class NodeAttribute {
+
+ public static NodeAttribute newInstance(String attributeName,
+ NodeAttributeType attributeType, String attributeValue) {
+ NodeAttribute nodeAttribute = Records.newRecord(NodeAttribute.class);
+ nodeAttribute.setAttributeName(attributeName);
+ nodeAttribute.setAttributeType(attributeType);
+ nodeAttribute.setAttributeValue(attributeValue);
+ return nodeAttribute;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getAttributeName();
+
+ @Public
+ @Unstable
+ public abstract void setAttributeName(String attributeName);
+
+ @Public
+ @Unstable
+ public abstract String getAttributeValue();
+
+ @Public
+ @Unstable
+ public abstract void setAttributeValue(String attributeValue);
+
+ @Public
+ @Unstable
+ public abstract NodeAttributeType getAttributeType();
+
+ @Public
+ @Unstable
+ public abstract void setAttributeType(NodeAttributeType attributeType);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java
new file mode 100644
index 0000000..3f281c8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttributeType.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p>
+ * Type of a <code>node Attribute</code>.
+ * </p>
+ * Based on this attribute expressions and values will be evaluated.
+ */
+@Public
+@Unstable
+public enum NodeAttributeType {
+ /** string type node attribute. */
+ STRING
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
index 8523342..58bb270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/ResourceManagerAdministrationProtocol.java
@@ -30,6 +30,8 @@ import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
@@ -37,6 +39,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRespons
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshQueuesResponse;
@@ -52,8 +56,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequ
import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.UpdateNodeResourceResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesResourcesResponse;
@Private
public interface ResourceManagerAdministrationProtocol extends GetUserMappingsProtocol {
@@ -144,4 +146,11 @@ public interface ResourceManagerAdministrationProtocol extends GetUserMappingsPr
public RefreshClusterMaxPriorityResponse refreshClusterMaxPriority(
RefreshClusterMaxPriorityRequest request) throws YarnException,
IOException;
+
+
+ @Private
+ @Idempotent
+ public NodesToAttributesMappingResponse mapAttributesToNodes(
+ NodesToAttributesMappingRequest request) throws YarnException,
+ IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java
new file mode 100644
index 0000000..5de1504
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/AttributeMappingOperationType.java
@@ -0,0 +1,42 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * <p>
+ * Type of node to attribute mapping operation.
+ * </p>
+ *
+ */
+@Public
+@Unstable
+public enum AttributeMappingOperationType {
+ /** Replaces the existing node to attribute mapping with new mapping.*/
+ REPLACE,
+
+ /** Add attribute(s) to a node and if it already exists will update the
+ * value.*/
+ ADD,
+
+ /** Removes attribute(s) mapped to a node. */
+ REMOVE
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java
new file mode 100644
index 0000000..b2e38b4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeToAttributes.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * Represents a mapping of Node id to list of attributes.
+ */
+@Public
+@Unstable
+public abstract class NodeToAttributes {
+
+ public static NodeToAttributes newInstance(String node,
+ List<NodeAttribute> attributes) {
+ NodeToAttributes nodeIdToAttributes =
+ Records.newRecord(NodeToAttributes.class);
+ nodeIdToAttributes.setNode(node);
+ nodeIdToAttributes.setNodeAttributes(attributes);
+ return nodeIdToAttributes;
+ }
+
+ @Public
+ @Unstable
+ public abstract String getNode();
+
+ @Public
+ @Unstable
+ public abstract void setNode(String node);
+
+ @Public
+ @Unstable
+ public abstract List<NodeAttribute> getNodeAttributes();
+
+ @Public
+ @Unstable
+ public abstract void setNodeAttributes(List<NodeAttribute> attributes);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java
new file mode 100644
index 0000000..71421ed
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingRequest.java
@@ -0,0 +1,69 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * list of node-attribute mapping request info.
+ */
+@Public
+@Unstable
+public abstract class NodesToAttributesMappingRequest {
+
+ public static NodesToAttributesMappingRequest newInstance(
+ AttributeMappingOperationType operation,
+ List<NodeToAttributes> nodesToAttributes, boolean failOnUnknownNodes) {
+ NodesToAttributesMappingRequest request =
+ Records.newRecord(NodesToAttributesMappingRequest.class);
+ request.setNodesToAttributes(nodesToAttributes);
+ request.setFailOnUnknownNodes(failOnUnknownNodes);
+ request.setOperation(operation);
+ return request;
+ }
+
+ @Public
+ @Unstable
+ public abstract void setNodesToAttributes(
+ List<NodeToAttributes> nodesToAttributes);
+
+ @Public
+ @Unstable
+ public abstract List<NodeToAttributes> getNodesToAttributes();
+
+ @Public
+ @Unstable
+ public abstract void setFailOnUnknownNodes(boolean failOnUnknownNodes);
+
+ @Public
+ @Unstable
+ public abstract boolean getFailOnUnknownNodes();
+
+ @Public
+ @Unstable
+ public abstract void setOperation(AttributeMappingOperationType operation);
+
+ @Public
+ @Unstable
+ public abstract AttributeMappingOperationType getOperation();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java
new file mode 100644
index 0000000..8e44adf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodesToAttributesMappingResponse.java
@@ -0,0 +1,27 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+import org.apache.hadoop.yarn.util.Records;
+
+public class NodesToAttributesMappingResponse {
+ public static NodesToAttributesMappingResponse newInstance() {
+ return Records.newRecord(NodesToAttributesMappingResponse.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
index 1134623..032aa8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/resourcemanager_administration_protocol.proto
@@ -45,4 +45,5 @@ service ResourceManagerAdministrationProtocolService {
rpc replaceLabelsOnNodes(ReplaceLabelsOnNodeRequestProto) returns (ReplaceLabelsOnNodeResponseProto);
rpc checkForDecommissioningNodes(CheckForDecommissioningNodesRequestProto) returns (CheckForDecommissioningNodesResponseProto);
rpc refreshClusterMaxPriority(RefreshClusterMaxPriorityRequestProto) returns (RefreshClusterMaxPriorityResponseProto);
+ rpc mapAttributesToNodes(NodesToAttributesMappingRequestProto) returns (NodesToAttributesMappingResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
index e8c92d9..5b93aec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/yarn_server_resourcemanager_service_protos.proto
@@ -130,6 +130,27 @@ enum DecommissionTypeProto {
GRACEFUL = 2;
FORCEFUL = 3;
}
+
+
+enum AttributeMappingOperationTypeProto {
+ REPLACE = 1;
+ ADD = 2;
+ REMOVE = 3;
+}
+
+message NodesToAttributesMappingRequestProto {
+ optional AttributeMappingOperationTypeProto operation = 1 [default = REPLACE];
+ repeated NodeToAttributesProto nodeToAttributes = 2;
+ optional bool failOnUnknownNodes = 3;
+}
+
+message NodeToAttributesProto {
+ optional string node = 1;
+ repeated NodeAttributeProto nodeAttributes = 2;
+}
+
+message NodesToAttributesMappingResponseProto {
+}
//////////////////////////////////////////////////////////////////
///////////// RM Failover related records ////////////////////////
//////////////////////////////////////////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5e200dc..46acffa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -371,6 +371,17 @@ message NodeLabelProto {
optional bool isExclusive = 2 [default = true];
}
+enum NodeAttributeTypeProto {
+ STRING = 1;
+}
+
+message NodeAttributeProto {
+ optional string attributeName = 1;
+ optional NodeAttributeTypeProto attributeType = 2;
+ optional string attributeValue = 3;
+}
+
+
enum ContainerTypeProto {
APPLICATION_MASTER = 1;
TASK = 2;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
new file mode 100644
index 0000000..11c9c48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
@@ -0,0 +1,155 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeTypeProto;
+
+public class NodeAttributePBImpl extends NodeAttribute {
+ private NodeAttributeProto proto = NodeAttributeProto.getDefaultInstance();
+ private NodeAttributeProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public NodeAttributePBImpl() {
+ builder = NodeAttributeProto.newBuilder();
+ }
+
+ public NodeAttributePBImpl(NodeAttributeProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public NodeAttributeProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = NodeAttributeProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public String getAttributeName() {
+ NodeAttributeProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAttributeName()) {
+ return null;
+ }
+ return p.getAttributeName();
+ }
+
+ @Override
+ public void setAttributeName(String attributeName) {
+ maybeInitBuilder();
+ builder.setAttributeName(attributeName);
+ }
+
+ @Override
+ public String getAttributeValue() {
+ NodeAttributeProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAttributeValue()) {
+ return null;
+ }
+ return p.getAttributeValue();
+ }
+
+ @Override
+ public void setAttributeValue(String attributeValue) {
+ maybeInitBuilder();
+ builder.setAttributeValue(attributeValue);
+ }
+
+ @Override
+ public NodeAttributeType getAttributeType() {
+ NodeAttributeProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAttributeType()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getAttributeType());
+ }
+
+ @Override
+ public void setAttributeType(NodeAttributeType attributeType) {
+ maybeInitBuilder();
+ if (attributeType == null) {
+ builder.clearAttributeType();
+ return;
+ }
+ builder.setAttributeType(convertToProtoFormat(attributeType));
+ }
+
+ private NodeAttributeTypeProto convertToProtoFormat(
+ NodeAttributeType attributeType) {
+ return NodeAttributeTypeProto.valueOf(attributeType.name());
+ }
+
+ private NodeAttributeType convertFromProtoFormat(
+ NodeAttributeTypeProto containerState) {
+ return NodeAttributeType.valueOf(containerState.name());
+ }
+
+ @Override
+ public String toString() {
+ return " name-" + getAttributeName() + ":value-" + getAttributeValue()
+ + ":type-" + getAttributeType();
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (obj instanceof NodeAttribute) {
+ NodeAttribute other = (NodeAttribute) obj;
+ if (!compare(getAttributeName(), other.getAttributeName())) {
+ return false;
+ }
+ if (!compare(getAttributeValue(), other.getAttributeValue())) {
+ return false;
+ }
+ if (!compare(getAttributeType(), other.getAttributeType())) {
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ private static boolean compare(Object left, Object right) {
+ if (left == null) {
+ return right == null;
+ } else {
+ return left.equals(right);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
index 077edf3..156ed05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/ResourceManagerAdministrationProtocolPBClientImpl.java
@@ -33,9 +33,11 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddTo
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshClusterMaxPriorityRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResourcesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshServiceAclsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshSuperUserGroupsConfigurationRequestProto;
@@ -43,13 +45,14 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Refre
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResourcesRequestProto;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
@@ -76,11 +79,15 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNod
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodesToAttributesMappingRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodesToAttributesMappingResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshClusterMaxPriorityRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshClusterMaxPriorityResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshQueuesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshQueuesResponsePBImpl;
@@ -96,8 +103,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOn
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesResponsePBImpl;
import com.google.protobuf.ServiceException;
@@ -323,4 +328,19 @@ public class ResourceManagerAdministrationProtocolPBClientImpl implements Resour
return null;
}
}
+
+ @Override
+ public NodesToAttributesMappingResponse mapAttributesToNodes(
+ NodesToAttributesMappingRequest request)
+ throws YarnException, IOException {
+ NodesToAttributesMappingRequestProto requestProto =
+ ((NodesToAttributesMappingRequestPBImpl) request).getProto();
+ try {
+ return new NodesToAttributesMappingResponsePBImpl(
+ proxy.mapAttributesToNodes(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
index aafce08..0036339 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/ResourceManagerAdministrationProtocolPBServiceImpl.java
@@ -28,11 +28,15 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Check
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.GetGroupsForUserResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshClusterMaxPriorityRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshClusterMaxPriorityResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResourcesRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResourcesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshQueuesResponseProto;
@@ -48,13 +52,13 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.Repla
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.UpdateNodeResourceResponseProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResourcesRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesResourcesResponseProto;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocolPB;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityResponse;
@@ -71,11 +75,15 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNod
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodesToAttributesMappingRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodesToAttributesMappingResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshClusterMaxPriorityRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshClusterMaxPriorityResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshQueuesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshQueuesResponsePBImpl;
@@ -91,8 +99,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOn
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.UpdateNodeResourceResponsePBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesResourcesResponsePBImpl;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -336,4 +342,21 @@ public class ResourceManagerAdministrationProtocolPBServiceImpl implements Resou
throw new ServiceException(e);
}
}
+
+ @Override
+ public NodesToAttributesMappingResponseProto mapAttributesToNodes(
+ RpcController controller, NodesToAttributesMappingRequestProto proto)
+ throws ServiceException {
+ NodesToAttributesMappingRequest request =
+ new NodesToAttributesMappingRequestPBImpl(proto);
+ try {
+ NodesToAttributesMappingResponse response =
+ real.mapAttributesToNodes(request);
+ return ((NodesToAttributesMappingResponsePBImpl) response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java
new file mode 100644
index 0000000..7b52d03
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeToAttributesPBImpl.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeToAttributesProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeToAttributesProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+
+public class NodeToAttributesPBImpl extends NodeToAttributes {
+ private NodeToAttributesProto proto =
+ NodeToAttributesProto.getDefaultInstance();
+ private NodeToAttributesProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List<NodeAttribute> nodeAttributes = null;
+
+ public NodeToAttributesPBImpl() {
+ builder = NodeToAttributesProto.newBuilder();
+ }
+
+ public NodeToAttributesPBImpl(NodeToAttributesProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ if (this.nodeAttributes != null) {
+ for (NodeAttribute nodeAttribute : nodeAttributes) {
+ builder.addNodeAttributes(
+ ((NodeAttributePBImpl) nodeAttribute).getProto());
+ }
+ }
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ public NodeToAttributesProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = NodeToAttributesProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public String getNode() {
+ NodeToAttributesProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNode()) {
+ return null;
+ }
+ return p.getNode();
+ }
+
+ @Override
+ public void setNode(String node) {
+ maybeInitBuilder();
+ builder.setNode(node);
+ }
+
+ private void initNodeAttributes() {
+ if (this.nodeAttributes != null) {
+ return;
+ }
+
+ NodeToAttributesProtoOrBuilder p = viaProto ? proto : builder;
+ List<NodeAttributeProto> nodeAttributesProtoList =
+ p.getNodeAttributesList();
+ List<NodeAttribute> attributes = new ArrayList<>();
+ if (nodeAttributesProtoList == null
+ || nodeAttributesProtoList.size() == 0) {
+ this.nodeAttributes = attributes;
+ return;
+ }
+ for (NodeAttributeProto nodeAttributeProto : nodeAttributesProtoList) {
+ attributes.add(new NodeAttributePBImpl(nodeAttributeProto));
+ }
+ this.nodeAttributes = attributes;
+ }
+
+ @Override
+ public List<NodeAttribute> getNodeAttributes() {
+ initNodeAttributes();
+ return this.nodeAttributes;
+ }
+
+ @Override
+ public void setNodeAttributes(List<NodeAttribute> attributes) {
+ if (nodeAttributes == null) {
+ nodeAttributes = new ArrayList<>();
+ }
+ nodeAttributes.clear();
+ nodeAttributes.addAll(attributes);
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (obj instanceof NodeToAttributes) {
+ NodeToAttributes other = (NodeToAttributes) obj;
+ if (getNodeAttributes() == null) {
+ if (other.getNodeAttributes() != null) {
+ return false;
+ }
+ } else if (!getNodeAttributes().containsAll(other.getNodeAttributes())) {
+ return false;
+ }
+
+ if (getNode() == null) {
+ if (other.getNode() != null) {
+ return false;
+ }
+ } else if (!getNode().equals(other.getNode())) {
+ return false;
+ }
+
+ return true;
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java
new file mode 100644
index 0000000..b319b26
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingRequestPBImpl.java
@@ -0,0 +1,194 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AttributeMappingOperationTypeProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeToAttributesProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+
+public class NodesToAttributesMappingRequestPBImpl
+ extends NodesToAttributesMappingRequest {
+ private NodesToAttributesMappingRequestProto proto =
+ NodesToAttributesMappingRequestProto.getDefaultInstance();
+ private NodesToAttributesMappingRequestProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ private List<NodeToAttributes> nodeAttributesMapping = null;
+
+ public NodesToAttributesMappingRequestPBImpl() {
+ builder = NodesToAttributesMappingRequestProto.newBuilder();
+ }
+
+ public NodesToAttributesMappingRequestPBImpl(
+ NodesToAttributesMappingRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ if (this.nodeAttributesMapping != null) {
+ for (NodeToAttributes nodeAttributes : nodeAttributesMapping) {
+ builder.addNodeToAttributes(
+ ((NodeToAttributesPBImpl) nodeAttributes).getProto());
+ }
+ }
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ public NodesToAttributesMappingRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = NodesToAttributesMappingRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public void setNodesToAttributes(List<NodeToAttributes> nodesToAttributes) {
+ if (nodeAttributesMapping == null) {
+ nodeAttributesMapping = new ArrayList<>();
+ }
+ if(nodesToAttributes == null) {
+ throw new IllegalArgumentException("nodesToAttributes cannot be null");
+ }
+ nodeAttributesMapping.clear();
+ nodeAttributesMapping.addAll(nodesToAttributes);
+ }
+
+ private void initNodeAttributesMapping() {
+ if (this.nodeAttributesMapping != null) {
+ return;
+ }
+
+ NodesToAttributesMappingRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ List<NodeToAttributesProto> nodeAttributesProtoList =
+ p.getNodeToAttributesList();
+ List<NodeToAttributes> attributes = new ArrayList<>();
+ if (nodeAttributesProtoList == null
+ || nodeAttributesProtoList.size() == 0) {
+ this.nodeAttributesMapping = attributes;
+ return;
+ }
+ for (NodeToAttributesProto nodeAttributeProto : nodeAttributesProtoList) {
+ attributes.add(new NodeToAttributesPBImpl(nodeAttributeProto));
+ }
+ this.nodeAttributesMapping = attributes;
+ }
+
+ @Override
+ public List<NodeToAttributes> getNodesToAttributes() {
+ initNodeAttributesMapping();
+ return this.nodeAttributesMapping;
+ }
+
+ @Override
+ public void setFailOnUnknownNodes(boolean failOnUnknownNodes) {
+ maybeInitBuilder();
+ builder.setFailOnUnknownNodes(failOnUnknownNodes);
+ }
+
+ @Override
+ public boolean getFailOnUnknownNodes() {
+ NodesToAttributesMappingRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ return p.getFailOnUnknownNodes();
+ }
+
+ @Override
+ public void setOperation(AttributeMappingOperationType operation) {
+ maybeInitBuilder();
+ builder.setOperation(convertToProtoFormat(operation));
+ }
+
+ private AttributeMappingOperationTypeProto convertToProtoFormat(
+ AttributeMappingOperationType operation) {
+ return AttributeMappingOperationTypeProto.valueOf(operation.name());
+ }
+
+ private AttributeMappingOperationType convertFromProtoFormat(
+ AttributeMappingOperationTypeProto operationTypeProto) {
+ return AttributeMappingOperationType.valueOf(operationTypeProto.name());
+ }
+
+ @Override
+ public AttributeMappingOperationType getOperation() {
+ NodesToAttributesMappingRequestProtoOrBuilder p =
+ viaProto ? proto : builder;
+ if (!p.hasOperation()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getOperation());
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (obj instanceof NodesToAttributesMappingRequest) {
+ NodesToAttributesMappingRequest other =
+ (NodesToAttributesMappingRequest) obj;
+ if (getNodesToAttributes() == null) {
+ if (other.getNodesToAttributes() != null) {
+ return false;
+ }
+ } else if (!getNodesToAttributes()
+ .containsAll(other.getNodesToAttributes())) {
+ return false;
+ }
+
+ if (getOperation() == null) {
+ if (other.getOperation() != null) {
+ return false;
+ }
+ } else if (!getOperation().equals(other.getOperation())) {
+ return false;
+ }
+
+ return getFailOnUnknownNodes() == other.getFailOnUnknownNodes();
+ }
+ return false;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java
new file mode 100644
index 0000000..0554ef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodesToAttributesMappingResponsePBImpl.java
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingResponseProto;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
+
+public class NodesToAttributesMappingResponsePBImpl
+ extends NodesToAttributesMappingResponse {
+
+ private NodesToAttributesMappingResponseProto proto =
+ NodesToAttributesMappingResponseProto.getDefaultInstance();
+ private NodesToAttributesMappingResponseProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public NodesToAttributesMappingResponsePBImpl() {
+ builder = NodesToAttributesMappingResponseProto.newBuilder();
+ }
+
+ public NodesToAttributesMappingResponsePBImpl(
+ NodesToAttributesMappingResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public NodesToAttributesMappingResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index dfa0c88..c4438ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -124,6 +124,7 @@ import org.apache.hadoop.yarn.api.records.ExecutionTypeRequest;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NMToken;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeReport;
@@ -176,6 +177,7 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ExecutionTypeRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
@@ -214,6 +216,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
@@ -238,6 +241,8 @@ import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddTo
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodeToAttributesProto;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.NodesToAttributesMappingRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshAdminAclsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RefreshNodesRequestProto;
@@ -320,10 +325,14 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseP
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.CheckForDecommissioningNodesResponsePBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodeToAttributesPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.NodesToAttributesMappingRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshAdminAclsResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RefreshNodesRequestPBImpl;
@@ -434,7 +443,12 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(ResourceSizing.class);
generateByNewInstance(SchedulingRequest.class);
generateByNewInstance(RejectedSchedulingRequest.class);
- }
+ //for Node attribute support
+ generateByNewInstance(NodeAttribute.class);
+ generateByNewInstance(NodeToAttributes.class);
+ generateByNewInstance(NodesToAttributesMappingRequest.class);
+
+ }
@Test
public void testAllocateRequestPBImpl() throws Exception {
@@ -1228,4 +1242,22 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
validatePBImplRecord(GetAllResourceTypeInfoResponsePBImpl.class,
YarnServiceProtos.GetAllResourceTypeInfoResponseProto.class);
}
+
+ @Test
+ public void testNodeAttributePBImpl() throws Exception {
+ validatePBImplRecord(NodeAttributePBImpl.class,
+ NodeAttributeProto.class);
+ }
+
+ @Test
+ public void testNodeToAttributesPBImpl() throws Exception {
+ validatePBImplRecord(NodeToAttributesPBImpl.class,
+ NodeToAttributesProto.class);
+ }
+
+ @Test
+ public void testNodesToAttributesMappingRequestPBImpl() throws Exception {
+ validatePBImplRecord(NodesToAttributesMappingRequestPBImpl.class,
+ NodesToAttributesMappingRequestProto.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
index 15e1cea..38bd2a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/MockResourceManagerFacade.java
@@ -163,6 +163,8 @@ import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import com.google.common.base.Strings;
@@ -180,15 +182,15 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
private HashSet<ApplicationId> applicationMap = new HashSet<>();
private HashSet<ApplicationId> keepContainerOnUams = new HashSet<>();
- private HashMap<ApplicationAttemptId, List<ContainerId>>
- applicationContainerIdMap = new HashMap<>();
+ private HashMap<ApplicationAttemptId, List<ContainerId>> applicationContainerIdMap =
+ new HashMap<ApplicationAttemptId, List<ContainerId>>();
private AtomicInteger containerIndex = new AtomicInteger(0);
private Configuration conf;
private int subClusterId;
final private AtomicInteger applicationCounter = new AtomicInteger(0);
// True if the Mock RM is running, false otherwise.
- // This property allows us to write tests for specific scenario as YARN RM
+ // This property allows us to write tests for specific scenario as Yarn RM
// down e.g. network issue, failover.
private boolean isRunning;
@@ -493,7 +495,7 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
if (request.getApplicationSubmissionContext().getUnmanagedAM()
|| request.getApplicationSubmissionContext()
.getKeepContainersAcrossApplicationAttempts()) {
- keepContainerOnUams.add(appId);
+ keepContainerOnUams.add(appId);
}
return SubmitApplicationResponse.newInstance();
}
@@ -511,7 +513,6 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
throw new ApplicationNotFoundException(
"Trying to kill an absent application: " + appId);
}
- keepContainerOnUams.remove(appId);
}
LOG.info("Force killing application: " + appId);
return KillApplicationResponse.newInstance(true);
@@ -899,4 +900,10 @@ public class MockResourceManagerFacade implements ApplicationClientProtocol,
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
return null;
}
+
+ @Override
+ public NodesToAttributesMappingResponse mapAttributesToNodes(NodesToAttributesMappingRequest request)
+ throws YarnException, IOException {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 3c117bc..bff3f2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -66,6 +66,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsR
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
@@ -961,4 +963,12 @@ public class AdminService extends CompositeService implements
rm.getRMContext().getScheduler().setClusterMaxPriority(conf);
}
+
+ @Override
+ public NodesToAttributesMappingResponse mapAttributesToNodes(
+ NodesToAttributesMappingRequest request)
+ throws YarnException, IOException {
+ return recordFactory
+ .newRecordInstance(NodesToAttributesMappingResponse.class);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
index 7e6a1ff..b4140c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/DefaultRMAdminRequestInterceptor.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsR
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
@@ -212,4 +214,11 @@ public class DefaultRMAdminRequestInterceptor
public String[] getGroupsForUser(String userName) throws IOException {
return rmAdminProxy.getGroupsForUser(userName);
}
+
+ @Override
+ public NodesToAttributesMappingResponse mapAttributesToNodes(
+ NodesToAttributesMappingRequest request)
+ throws YarnException, IOException {
+ return rmAdminProxy.mapAttributesToNodes(request);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
index b8b7ad8..8d83b23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/rmadmin/RouterRMAdminService.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsR
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
@@ -420,4 +422,12 @@ public class RouterRMAdminService extends AbstractService
RequestInterceptorChainWrapper pipeline = getInterceptorChain();
return pipeline.getRootInterceptor().refreshClusterMaxPriority(request);
}
+
+ @Override
+ public NodesToAttributesMappingResponse mapAttributesToNodes(
+ NodesToAttributesMappingRequest request)
+ throws YarnException, IOException {
+ RequestInterceptorChainWrapper pipeline = getInterceptorChain();
+ return pipeline.getRootInterceptor().mapAttributesToNodes(request);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/461dcea8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
index 38dcc3d..4d6a3ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/rmadmin/PassThroughRMAdminRequestInterceptor.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsR
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
@@ -145,4 +147,11 @@ public class PassThroughRMAdminRequestInterceptor
return getNextInterceptor().getGroupsForUser(user);
}
+ @Override
+ public NodesToAttributesMappingResponse mapAttributesToNodes(
+ NodesToAttributesMappingRequest request)
+ throws YarnException, IOException {
+ return getNextInterceptor().mapAttributesToNodes(request);
+ }
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: HADOOP-15320. Remove customized
getFileBlockLocations for hadoop-azure and hadoop-azure-datalake. Contributed
by Shanyu Zhao
Posted by na...@apache.org.
HADOOP-15320. Remove customized getFileBlockLocations for hadoop-azure and hadoop-azure-datalake. Contributed by Shanyu Zhao
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/081c3501
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/081c3501
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/081c3501
Branch: refs/heads/YARN-3409
Commit: 081c3501885c543bb1f159929d456d1ba2e3650c
Parents: 0b1c2b5
Author: Chris Douglas <cd...@apache.org>
Authored: Wed Mar 28 11:58:59 2018 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Wed Mar 28 11:58:59 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/fs/adl/AdlFileSystem.java | 40 ------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 46 ------
...TestNativeAzureFileSystemBlockLocations.java | 141 -------------------
3 files changed, 227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/081c3501/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
index 9f54a36..aa6babe 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/main/java/org/apache/hadoop/fs/adl/AdlFileSystem.java
@@ -46,7 +46,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.ContentSummary.Builder;
import org.apache.hadoop.fs.CreateFlag;
@@ -910,45 +909,6 @@ public class AdlFileSystem extends FileSystem {
return ADL_BLOCK_SIZE;
}
- @Override
- public BlockLocation[] getFileBlockLocations(final FileStatus status,
- final long offset, final long length) throws IOException {
- if (status == null) {
- return null;
- }
-
- if ((offset < 0) || (length < 0)) {
- throw new IllegalArgumentException("Invalid start or len parameter");
- }
-
- if (status.getLen() < offset) {
- return new BlockLocation[0];
- }
-
- final String[] name = {"localhost"};
- final String[] host = {"localhost"};
- long blockSize = ADL_BLOCK_SIZE;
- int numberOfLocations =
- (int) (length / blockSize) + ((length % blockSize == 0) ? 0 : 1);
- BlockLocation[] locations = new BlockLocation[numberOfLocations];
- for (int i = 0; i < locations.length; i++) {
- long currentOffset = offset + (i * blockSize);
- long currentLength = Math.min(blockSize, offset + length - currentOffset);
- locations[i] = new BlockLocation(name, host, currentOffset,
- currentLength);
- }
-
- return locations;
- }
-
- @Override
- public BlockLocation[] getFileBlockLocations(final Path p, final long offset,
- final long length) throws IOException {
- // read ops incremented in getFileStatus
- FileStatus fileStatus = getFileStatus(p);
- return getFileBlockLocations(fileStatus, offset, length);
- }
-
/**
* Get replication.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/081c3501/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 3d44b20..e05327e 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -52,7 +52,6 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.BufferedFSInputStream;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -726,10 +725,6 @@ public class NativeAzureFileSystem extends FileSystem {
static final String AZURE_CHMOD_USERLIST_PROPERTY_DEFAULT_VALUE = "*";
- static final String AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME =
- "fs.azure.block.location.impersonatedhost";
- private static final String AZURE_BLOCK_LOCATION_HOST_DEFAULT =
- "localhost";
static final String AZURE_RINGBUFFER_CAPACITY_PROPERTY_NAME =
"fs.azure.ring.buffer.capacity";
static final String AZURE_OUTPUT_STREAM_BUFFER_SIZE_PROPERTY_NAME =
@@ -3470,47 +3465,6 @@ public class NativeAzureFileSystem extends FileSystem {
}
/**
- * Return an array containing hostnames, offset and size of
- * portions of the given file. For WASB we'll just lie and give
- * fake hosts to make sure we get many splits in MR jobs.
- */
- @Override
- public BlockLocation[] getFileBlockLocations(FileStatus file,
- long start, long len) throws IOException {
- if (file == null) {
- return null;
- }
-
- if ((start < 0) || (len < 0)) {
- throw new IllegalArgumentException("Invalid start or len parameter");
- }
-
- if (file.getLen() < start) {
- return new BlockLocation[0];
- }
- final String blobLocationHost = getConf().get(
- AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME,
- AZURE_BLOCK_LOCATION_HOST_DEFAULT);
- final String[] name = { blobLocationHost };
- final String[] host = { blobLocationHost };
- long blockSize = file.getBlockSize();
- if (blockSize <= 0) {
- throw new IllegalArgumentException(
- "The block size for the given file is not a positive number: "
- + blockSize);
- }
- int numberOfLocations = (int) (len / blockSize)
- + ((len % blockSize == 0) ? 0 : 1);
- BlockLocation[] locations = new BlockLocation[numberOfLocations];
- for (int i = 0; i < locations.length; i++) {
- long currentOffset = start + (i * blockSize);
- long currentLength = Math.min(blockSize, start + len - currentOffset);
- locations[i] = new BlockLocation(name, host, currentOffset, currentLength);
- }
- return locations;
- }
-
- /**
* Set the working directory to the given directory.
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/081c3501/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java
deleted file mode 100644
index b280cac..0000000
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestNativeAzureFileSystemBlockLocations.java
+++ /dev/null
@@ -1,141 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.fs.azure;
-
-import java.io.OutputStream;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.BlockLocation;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.junit.Test;
-
-/**
- * Test block location logic.
- */
-public class TestNativeAzureFileSystemBlockLocations
- extends AbstractWasbTestWithTimeout {
- @Test
- public void testNumberOfBlocks() throws Exception {
- Configuration conf = new Configuration();
- conf.set(NativeAzureFileSystem.AZURE_BLOCK_SIZE_PROPERTY_NAME, "500");
- AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount
- .createMock(conf);
- FileSystem fs = testAccount.getFileSystem();
- Path testFile = createTestFile(fs, 1200);
- FileStatus stat = fs.getFileStatus(testFile);
- assertEquals(500, stat.getBlockSize());
- testAccount.cleanup();
- }
-
- @Test
- public void testBlockLocationsTypical() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(210, 50, 0, 210);
- assertEquals(5, locations.length);
- assertEquals("localhost", locations[0].getHosts()[0]);
- assertEquals(50, locations[0].getLength());
- assertEquals(10, locations[4].getLength());
- assertEquals(100, locations[2].getOffset());
- }
-
- @Test
- public void testBlockLocationsEmptyFile() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(0, 50, 0, 0);
- assertEquals(0, locations.length);
- }
-
- @Test
- public void testBlockLocationsSmallFile() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(1, 50, 0, 1);
- assertEquals(1, locations.length);
- assertEquals(1, locations[0].getLength());
- }
-
- @Test
- public void testBlockLocationsExactBlockSizeMultiple() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(200, 50, 0, 200);
- assertEquals(4, locations.length);
- assertEquals(150, locations[3].getOffset());
- assertEquals(50, locations[3].getLength());
- }
-
- @Test
- public void testBlockLocationsSubsetOfFile() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(205, 10, 15, 35);
- assertEquals(4, locations.length);
- assertEquals(10, locations[0].getLength());
- assertEquals(15, locations[0].getOffset());
- assertEquals(5, locations[3].getLength());
- assertEquals(45, locations[3].getOffset());
- }
-
- @Test
- public void testBlockLocationsOutOfRangeSubsetOfFile() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(205, 10, 300, 10);
- assertEquals(0, locations.length);
- }
-
- @Test
- public void testBlockLocationsEmptySubsetOfFile() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(205, 10, 0, 0);
- assertEquals(0, locations.length);
- }
-
- @Test
- public void testBlockLocationsDifferentLocationHost() throws Exception {
- BlockLocation[] locations = getBlockLocationsOutput(100, 10, 0, 100,
- "myblobhost");
- assertEquals(10, locations.length);
- assertEquals("myblobhost", locations[0].getHosts()[0]);
- }
-
- private static BlockLocation[] getBlockLocationsOutput(int fileSize,
- int blockSize, long start, long len) throws Exception {
- return getBlockLocationsOutput(fileSize, blockSize, start, len, null);
- }
-
- private static BlockLocation[] getBlockLocationsOutput(int fileSize,
- int blockSize, long start, long len, String blockLocationHost)
- throws Exception {
- Configuration conf = new Configuration();
- conf.set(NativeAzureFileSystem.AZURE_BLOCK_SIZE_PROPERTY_NAME, ""
- + blockSize);
- if (blockLocationHost != null) {
- conf.set(NativeAzureFileSystem.AZURE_BLOCK_LOCATION_HOST_PROPERTY_NAME,
- blockLocationHost);
- }
- AzureBlobStorageTestAccount testAccount = AzureBlobStorageTestAccount
- .createMock(conf);
- FileSystem fs = testAccount.getFileSystem();
- Path testFile = createTestFile(fs, fileSize);
- FileStatus stat = fs.getFileStatus(testFile);
- BlockLocation[] locations = fs.getFileBlockLocations(stat, start, len);
- testAccount.cleanup();
- return locations;
- }
-
- private static Path createTestFile(FileSystem fs, int size) throws Exception {
- Path testFile = new Path("/testFile");
- OutputStream outputStream = fs.create(testFile);
- outputStream.write(new byte[size]);
- outputStream.close();
- return testFile;
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-7497. Add file system based
scheduler configuration store. Contributed by Jiandan Yang.
Posted by na...@apache.org.
YARN-7497. Add file system based scheduler configuration store. Contributed by Jiandan Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1606dad1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1606dad1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1606dad1
Branch: refs/heads/YARN-3409
Commit: 1606dad133de0dbee59175509a22994ec570ea41
Parents: 6542d17
Author: Weiwei Yang <ww...@apache.org>
Authored: Fri Mar 30 21:41:33 2018 +0800
Committer: Weiwei Yang <ww...@apache.org>
Committed: Fri Mar 30 21:41:33 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 16 +-
.../src/main/resources/yarn-default.xml | 22 ++
.../scheduler/capacity/CapacityScheduler.java | 1 +
.../conf/FSSchedulerConfigurationStore.java | 303 +++++++++++++++++++
.../conf/MutableCSConfigurationProvider.java | 3 +
.../capacity/conf/YarnConfigurationStore.java | 2 +-
.../conf/TestFSSchedulerConfigurationStore.java | 173 +++++++++++
.../TestMutableCSConfigurationProvider.java | 50 +++
8 files changed, 568 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1f62bbd..42f2cae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -776,12 +776,15 @@ public class YarnConfiguration extends Configuration {
public static final String MEMORY_CONFIGURATION_STORE = "memory";
@Private
@Unstable
- public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+ public static final String FS_CONFIGURATION_STORE = "fs";
@Private
@Unstable
public static final String ZK_CONFIGURATION_STORE = "zk";
@Private
@Unstable
+ public static final String LEVELDB_CONFIGURATION_STORE = "leveldb";
+ @Private
+ @Unstable
public static final String DEFAULT_CONFIGURATION_STORE =
FILE_CONFIGURATION_STORE;
@Private
@@ -809,6 +812,17 @@ public class YarnConfiguration extends Configuration {
@Private
@Unstable
public static final long DEFAULT_RM_SCHEDCONF_ZK_MAX_LOGS = 1000;
+ @Private
+ @Unstable
+ public static final String SCHEDULER_CONFIGURATION_FS_PATH =
+ YARN_PREFIX + "scheduler.configuration.fs.path";
+ @Private
+ @Unstable
+ public static final String SCHEDULER_CONFIGURATION_FS_MAX_VERSION =
+ YARN_PREFIX + "scheduler.configuration.max.version";
+ @Private
+ @Unstable
+ public static final int DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION = 100;
/** Parent znode path under which ZKConfigurationStore will create znodes. */
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 114ba4b..81b6658 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3570,6 +3570,28 @@
<property>
<description>
+ The file system directory to store the configuration files. The path
+ can be any format as long as it follows hadoop compatible schema,
+ for example value "file:///path/to/dir" means to store files on local
+ file system, value "hdfs:///path/to/dir" means to store files on HDFS.
+ If resource manager HA is enabled, recommended to use hdfs schema so
+ it works in fail-over scenario.
+ </description>
+ <name>yarn.scheduler.configuration.fs.path</name>
+ <value>file://${hadoop.tmp.dir}/yarn/system/schedconf</value>
+ </property>
+
+ <property>
+ <description>
+ The max number of configuration file in filesystem.
+ Default is 100 for either.
+ </description>
+ <name>yarn.scheduler.configuration.max.version</name>
+ <value>100</value>
+ </property>
+
+ <property>
+ <description>
ZK root node path for configuration store when using zookeeper-based
configuration store.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index bf674a8..e59bdde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -326,6 +326,7 @@ public class CapacityScheduler extends
case YarnConfiguration.MEMORY_CONFIGURATION_STORE:
case YarnConfiguration.LEVELDB_CONFIGURATION_STORE:
case YarnConfiguration.ZK_CONFIGURATION_STORE:
+ case YarnConfiguration.FS_CONFIGURATION_STORE:
this.csConfProvider = new MutableCSConfigurationProvider(rmContext);
break;
default:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
new file mode 100644
index 0000000..2a24887
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.gson.GsonBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+
+
+/**
+ * A filesystem implementation of {@link YarnConfigurationStore}. Offer
+ * configuration storage in FileSystem
+ */
+public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
+ public static final Log LOG = LogFactory.getLog(
+ FSSchedulerConfigurationStore.class);
+
+ @VisibleForTesting
+ protected static final Version CURRENT_VERSION_INFO
+ = Version.newInstance(0, 1);
+
+ private static final String TMP = ".tmp";
+
+ private int maxVersion;
+ private Path schedulerConfDir;
+ private FileSystem fileSystem;
+ private LogMutation pendingMutation;
+ private PathFilter configFilePathFilter;
+ private volatile Configuration schedConf;
+ private volatile Configuration oldConf;
+ private Path tempConfigPath;
+
+ @Override
+ public void initialize(Configuration conf, Configuration vSchedConf,
+ RMContext rmContext) throws Exception {
+ this.configFilePathFilter = new PathFilter() {
+ @Override
+ public boolean accept(Path path) {
+ if (path == null) {
+ return false;
+ }
+ String pathName = path.getName();
+ return pathName.startsWith(YarnConfiguration.CS_CONFIGURATION_FILE)
+ && !pathName.endsWith(TMP);
+ }
+ };
+
+ String schedulerConfPathStr = conf.get(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH);
+ if (schedulerConfPathStr == null || schedulerConfPathStr.isEmpty()) {
+ throw new IOException(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH
+ + " must be set");
+ }
+ this.schedulerConfDir = new Path(schedulerConfPathStr);
+ this.fileSystem = this.schedulerConfDir.getFileSystem(conf);
+ this.maxVersion = conf.getInt(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION,
+ YarnConfiguration.DEFAULT_SCHEDULER_CONFIGURATION_FS_MAX_VERSION);
+ LOG.info("schedulerConfDir=" + schedulerConfPathStr);
+ LOG.info("capacity scheduler file max version = " + maxVersion);
+
+ if (!fileSystem.exists(schedulerConfDir)) {
+ if (!fileSystem.mkdirs(schedulerConfDir)) {
+ throw new IOException("mkdir " + schedulerConfPathStr + " failed");
+ }
+ }
+
+ // create capacity-schedule.xml.ts file if not existing
+ if (this.getConfigFileInputStream() == null) {
+ writeConfigurationToFileSystem(vSchedConf);
+ }
+
+ this.schedConf = this.getConfigurationFromFileSystem();
+ }
+
+ /**
+ * Update and persist latest configuration in temp file.
+ * @param logMutation configuration change to be persisted in write ahead log
+ * @throws IOException throw IOE when write temp configuration file fail
+ */
+ @Override
+ public void logMutation(LogMutation logMutation) throws IOException {
+ pendingMutation = logMutation;
+ LOG.info(new GsonBuilder().serializeNulls().create().toJson(logMutation));
+ oldConf = new Configuration(schedConf);
+ Map<String, String> mutations = pendingMutation.getUpdates();
+ for (Map.Entry<String, String> kv : mutations.entrySet()) {
+ if (kv.getValue() == null) {
+ this.schedConf.unset(kv.getKey());
+ } else {
+ this.schedConf.set(kv.getKey(), kv.getValue());
+ }
+ }
+ tempConfigPath = writeTmpConfig(schedConf);
+ }
+
+ /**
+ * @param isValid if true, finalize temp configuration file
+ * if false, remove temp configuration file and rollback
+ * @throws Exception throw IOE when write temp configuration file fail
+ */
+ @Override
+ public void confirmMutation(boolean isValid) throws Exception {
+ if (pendingMutation == null || tempConfigPath == null) {
+ LOG.warn("pendingMutation or tempConfigPath is null, do nothing");
+ return;
+ }
+ if (isValid) {
+ finalizeFileSystemFile();
+ } else {
+ schedConf = oldConf;
+ removeTmpConfigFile();
+ }
+ tempConfigPath = null;
+ }
+
+ private void finalizeFileSystemFile() throws IOException {
+ // call confirmMutation() make sure tempConfigPath is not null
+ Path finalConfigPath = getFinalConfigPath(tempConfigPath);
+ fileSystem.rename(tempConfigPath, finalConfigPath);
+ LOG.info("finalize temp configuration file successfully, finalConfigPath="
+ + finalConfigPath);
+ }
+
+ private Path getFinalConfigPath(Path tempPath) {
+ String tempConfigPathStr = tempPath.getName();
+ if (!tempConfigPathStr.endsWith(TMP)) {
+ LOG.warn(tempPath + " does not end with '"
+ + TMP + "' return null");
+ return null;
+ }
+ String finalConfigPathStr = tempConfigPathStr.substring(0,
+ (tempConfigPathStr.length() - TMP.length()));
+ return new Path(tempPath.getParent(), finalConfigPathStr);
+ }
+
+ private void removeTmpConfigFile() throws IOException {
+ // call confirmMutation() make sure tempConfigPath is not null
+ fileSystem.delete(tempConfigPath, true);
+ LOG.info("delete temp configuration file: " + tempConfigPath);
+ }
+
+ private Configuration getConfigurationFromFileSystem() throws IOException {
+ long start = Time.monotonicNow();
+
+ Configuration conf = new Configuration(false);
+ InputStream configInputStream = getConfigFileInputStream();
+ if (configInputStream == null) {
+ throw new IOException(
+ "no capacity scheduler file in " + this.schedulerConfDir);
+ }
+
+ conf.addResource(configInputStream);
+ Configuration result = new Configuration(false);
+ for (Map.Entry<String, String> entry : conf) {
+ result.set(entry.getKey(), entry.getValue());
+ }
+ LOG.info("upload conf from fileSystem took "
+ + (Time.monotonicNow() - start) + " ms");
+
+ //for ha transition, local schedConf may be old one.
+ this.schedConf = result;
+ return result;
+ }
+
+ private InputStream getConfigFileInputStream() throws IOException {
+ Path lastestConfigPath = getLatestConfigPath();
+ if (lastestConfigPath == null) {
+ return null;
+ }
+ return fileSystem.open(lastestConfigPath);
+ }
+
+ private Path getLatestConfigPath() throws IOException {
+ FileStatus[] fileStatuses = fileSystem.listStatus(this.schedulerConfDir,
+ this.configFilePathFilter);
+
+ if (fileStatuses == null || fileStatuses.length == 0) {
+ return null;
+ }
+ Arrays.sort(fileStatuses);
+
+ return fileStatuses[fileStatuses.length - 1].getPath();
+ }
+
+ @VisibleForTesting
+ private Path writeTmpConfig(Configuration vSchedConf) throws IOException {
+ long start = Time.monotonicNow();
+ String tempSchedulerConfigFile = YarnConfiguration.CS_CONFIGURATION_FILE
+ + "." + System.currentTimeMillis() + TMP;
+
+ Path tempSchedulerConfigPath = new Path(this.schedulerConfDir,
+ tempSchedulerConfigFile);
+
+ try (FSDataOutputStream outputStream = fileSystem.create(
+ tempSchedulerConfigPath)) {
+ //clean configuration file when num exceed maxVersion
+ cleanConfigurationFile();
+
+ vSchedConf.writeXml(outputStream);
+ LOG.info(
+ "write temp capacity configuration successfully, schedulerConfigFile="
+ + tempSchedulerConfigPath);
+ } catch (IOException e) {
+ LOG.info("write temp capacity configuration fail, schedulerConfigFile="
+ + tempSchedulerConfigPath, e);
+ throw e;
+ }
+ LOG.info("write temp configuration to fileSystem took "
+ + (Time.monotonicNow() - start) + " ms");
+ return tempSchedulerConfigPath;
+ }
+
+ @VisibleForTesting
+ void writeConfigurationToFileSystem(Configuration vSchedConf)
+ throws IOException {
+ tempConfigPath = writeTmpConfig(vSchedConf);
+ finalizeFileSystemFile();
+ }
+
+ private void cleanConfigurationFile() throws IOException {
+ FileStatus[] fileStatuses = fileSystem.listStatus(this.schedulerConfDir,
+ this.configFilePathFilter);
+
+ if (fileStatuses == null || fileStatuses.length <= this.maxVersion) {
+ return;
+ }
+ Arrays.sort(fileStatuses);
+ int configFileNum = fileStatuses.length;
+ if (fileStatuses.length > this.maxVersion) {
+ for (int i = 0; i < configFileNum - this.maxVersion; i++) {
+ fileSystem.delete(fileStatuses[i].getPath(), false);
+ LOG.info("delete config file " + fileStatuses[i].getPath());
+ }
+ }
+ }
+
+ @Override
+ public Configuration retrieve() throws IOException {
+ return getConfigurationFromFileSystem();
+ }
+
+ @Override
+ public List<LogMutation> getConfirmedConfHistory(long fromId) {
+ // Unimplemented.
+ return null;
+ }
+
+ @Override
+ protected Version getConfStoreVersion() throws Exception {
+ return null;
+ }
+
+ @Override
+ protected void storeVersion() throws Exception {
+
+ }
+
+ @Override
+ protected Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
+ public void close() throws IOException {
+ if (fileSystem != null) {
+ fileSystem.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index 40a19a4..9c3bf9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -77,6 +77,9 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
case YarnConfiguration.ZK_CONFIGURATION_STORE:
this.confStore = new ZKConfigurationStore();
break;
+ case YarnConfiguration.FS_CONFIGURATION_STORE:
+ this.confStore = new FSSchedulerConfigurationStore();
+ break;
default:
this.confStore = YarnConfigurationStoreFactory.getStore(config);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index 7fb52fc..ef0a44b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -123,7 +123,7 @@ public abstract class YarnConfigurationStore {
* Retrieve the persisted configuration.
* @return configuration as key-value
*/
- public abstract Configuration retrieve();
+ public abstract Configuration retrieve() throws IOException;
/**
* Get a list of confirmed configuration mutations starting from a given id.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
new file mode 100644
index 0000000..65314be
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestFSSchedulerConfigurationStore.java
@@ -0,0 +1,173 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.YarnConfigurationStore.LogMutation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+/**
+ * Tests {@link FSSchedulerConfigurationStore}.
+ */
+public class TestFSSchedulerConfigurationStore {
+ private FSSchedulerConfigurationStore configurationStore;
+ private Configuration conf;
+ private File testSchedulerConfigurationDir;
+
+ @Before
+ public void setUp() throws Exception {
+ configurationStore = new FSSchedulerConfigurationStore();
+ testSchedulerConfigurationDir = new File(
+ TestFSSchedulerConfigurationStore.class.getResource("").getPath()
+ + FSSchedulerConfigurationStore.class.getSimpleName());
+ testSchedulerConfigurationDir.mkdirs();
+
+ conf = new Configuration();
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH,
+ testSchedulerConfigurationDir.getAbsolutePath());
+ }
+
+ private void writeConf(Configuration config) throws IOException {
+ FileSystem fileSystem = FileSystem.get(new Configuration(config));
+ String schedulerConfigurationFile = YarnConfiguration.CS_CONFIGURATION_FILE
+ + "." + System.currentTimeMillis();
+ FSDataOutputStream outputStream = fileSystem.create(
+ new Path(testSchedulerConfigurationDir.getAbsolutePath(),
+ schedulerConfigurationFile));
+ config.writeXml(outputStream);
+ outputStream.close();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ FileUtils.deleteDirectory(testSchedulerConfigurationDir);
+ }
+
+ @Test
+ public void confirmMutationWithValid() throws Exception {
+ conf.setInt(
+ YarnConfiguration.SCHEDULER_CONFIGURATION_FS_MAX_VERSION, 2);
+ conf.set("a", "a");
+ conf.set("b", "b");
+ conf.set("c", "c");
+ writeConf(conf);
+ configurationStore.initialize(conf, conf, null);
+ Configuration storeConf = configurationStore.retrieve();
+ compareConfig(conf, storeConf);
+
+ Map<String, String> updates = new HashMap<>();
+ updates.put("a", null);
+ updates.put("b", "bb");
+
+ Configuration expectConfig = new Configuration(conf);
+ expectConfig.unset("a");
+ expectConfig.set("b", "bb");
+
+ LogMutation logMutation = new LogMutation(updates, "test");
+ configurationStore.logMutation(logMutation);
+ configurationStore.confirmMutation(true);
+ storeConf = configurationStore.retrieve();
+ assertEquals(null, storeConf.get("a"));
+ assertEquals("bb", storeConf.get("b"));
+ assertEquals("c", storeConf.get("c"));
+
+ compareConfig(expectConfig, storeConf);
+
+ updates.put("b", "bbb");
+ configurationStore.logMutation(logMutation);
+ configurationStore.confirmMutation(true);
+ storeConf = configurationStore.retrieve();
+ assertEquals(null, storeConf.get("a"));
+ assertEquals("bbb", storeConf.get("b"));
+ assertEquals("c", storeConf.get("c"));
+ }
+
+ @Test
+ public void confirmMutationWithInValid() throws Exception {
+ conf.set("a", "a");
+ conf.set("b", "b");
+ conf.set("c", "c");
+ writeConf(conf);
+ configurationStore.initialize(conf, conf, null);
+ Configuration storeConf = configurationStore.retrieve();
+ compareConfig(conf, storeConf);
+
+ Map<String, String> updates = new HashMap<>();
+ updates.put("a", null);
+ updates.put("b", "bb");
+
+ LogMutation logMutation = new LogMutation(updates, "test");
+ configurationStore.logMutation(logMutation);
+ configurationStore.confirmMutation(false);
+ storeConf = configurationStore.retrieve();
+
+ compareConfig(conf, storeConf);
+ }
+
+ @Test
+ public void retrieve() throws Exception {
+ Configuration schedulerConf = new Configuration();
+ schedulerConf.set("a", "a");
+ schedulerConf.setLong("long", 1L);
+ schedulerConf.setBoolean("boolean", true);
+ writeConf(schedulerConf);
+
+ configurationStore.initialize(conf, conf, null);
+ Configuration storedConfig = configurationStore.retrieve();
+
+ compareConfig(schedulerConf, storedConfig);
+ }
+
+ @Test
+ public void checkVersion() {
+ try {
+ configurationStore.checkVersion();
+ } catch (Exception e) {
+ fail("checkVersion throw exception");
+ }
+ }
+
+ private void compareConfig(Configuration schedulerConf,
+ Configuration storedConfig) {
+ for (Map.Entry<String, String> entry : schedulerConf) {
+ assertEquals(entry.getKey(), schedulerConf.get(entry.getKey()),
+ storedConfig.get(entry.getKey()));
+ }
+
+ for (Map.Entry<String, String> entry : storedConfig) {
+ assertEquals(entry.getKey(), storedConfig.get(entry.getKey()),
+ schedulerConf.get(entry.getKey()));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1606dad1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
index 5d43ebb..81bc7a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestMutableCSConfigurationProvider.java
@@ -18,7 +18,11 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
@@ -30,6 +34,8 @@ import org.apache.hadoop.yarn.webapp.dao.SchedConfUpdateInfo;
import org.junit.Before;
import org.junit.Test;
+import java.io.File;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -99,4 +105,48 @@ public class TestMutableCSConfigurationProvider {
assertNull(confProvider.loadConfiguration(conf).get(
"yarn.scheduler.capacity.root.a.badKey"));
}
+
+ @Test
+ public void testHDFSBackedProvider() throws Exception {
+ File testSchedulerConfigurationDir = new File(
+ TestMutableCSConfigurationProvider.class.getResource("").getPath()
+ + TestMutableCSConfigurationProvider.class.getSimpleName());
+ FileUtils.deleteDirectory(testSchedulerConfigurationDir);
+ testSchedulerConfigurationDir.mkdirs();
+
+ Configuration conf = new Configuration(false);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_STORE_CLASS,
+ YarnConfiguration.FS_CONFIGURATION_STORE);
+ conf.set(YarnConfiguration.SCHEDULER_CONFIGURATION_FS_PATH,
+ testSchedulerConfigurationDir.getAbsolutePath());
+ writeConf(conf, testSchedulerConfigurationDir.getAbsolutePath());
+
+ confProvider.init(conf);
+ assertNull(confProvider.loadConfiguration(conf)
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
+
+ confProvider.logAndApplyMutation(TEST_USER, goodUpdate);
+ confProvider.confirmPendingMutation(true);
+ assertEquals("goodVal", confProvider.loadConfiguration(conf)
+ .get("yarn.scheduler.capacity.root.a.goodKey"));
+
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
+ confProvider.logAndApplyMutation(TEST_USER, badUpdate);
+ confProvider.confirmPendingMutation(false);
+ assertNull(confProvider.loadConfiguration(conf).get(
+ "yarn.scheduler.capacity.root.a.badKey"));
+
+ }
+
+ private void writeConf(Configuration conf, String storePath)
+ throws IOException {
+ FileSystem fileSystem = FileSystem.get(new Configuration(conf));
+ String schedulerConfigurationFile = YarnConfiguration.CS_CONFIGURATION_FILE
+ + "." + System.currentTimeMillis();
+ try (FSDataOutputStream outputStream = fileSystem.create(
+ new Path(storePath, schedulerConfigurationFile))) {
+ conf.writeXml(outputStream);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-7988. Refactor
FSNodeLabelStore code for Node Attributes store support. Contributed by Bibin
A Chundatt.
Posted by na...@apache.org.
YARN-7988. Refactor FSNodeLabelStore code for Node Attributes store support. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a247908
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a247908
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a247908
Branch: refs/heads/YARN-3409
Commit: 3a24790847f940713b2f623b4d028e403fb85524
Parents: 5bd044e
Author: Sunil G <su...@apache.org>
Authored: Thu Mar 29 17:12:08 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 11:43:16 2018 +0800
----------------------------------------------------------------------
.../nodelabels/CommonNodeLabelsManager.java | 8 +-
.../nodelabels/FileSystemNodeLabelsStore.java | 268 +++----------------
.../hadoop/yarn/nodelabels/NodeLabelsStore.java | 32 +--
.../NonAppendableFSNodeLabelStore.java | 46 ++--
.../nodelabels/store/AbstractFSNodeStore.java | 213 +++++++++++++++
.../yarn/nodelabels/store/FSStoreOpHandler.java | 108 ++++++++
.../hadoop/yarn/nodelabels/store/StoreOp.java | 49 ++++
.../nodelabels/store/op/AddClusterLabelOp.java | 73 +++++
.../nodelabels/store/op/FSNodeStoreLogOp.java | 35 +++
.../nodelabels/store/op/NodeLabelMirrorOp.java | 82 ++++++
.../yarn/nodelabels/store/op/NodeToLabelOp.java | 75 ++++++
.../store/op/RemoveClusterLabelOp.java | 75 ++++++
.../yarn/nodelabels/store/package-info.java | 21 ++
...emoveFromClusterNodeLabelsRequestPBImpl.java | 21 +-
.../DummyCommonNodeLabelsManager.java | 8 +-
.../TestFileSystemNodeLabelsStore.java | 16 +-
.../nodelabels/NullRMNodeLabelsManager.java | 7 +
.../webapp/TestRMWebServicesNodeLabels.java | 2 +-
18 files changed, 845 insertions(+), 294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 24ee028..4973611 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -234,7 +234,10 @@ public class CommonNodeLabelsManager extends AbstractService {
return initNodeLabelStoreInProgress;
}
- boolean isCentralizedConfiguration() {
+ /**
+ * @return true if node label configuration type is not distributed.
+ */
+ public boolean isCentralizedConfiguration() {
return isCentralizedNodeLabelConfiguration;
}
@@ -245,8 +248,7 @@ public class CommonNodeLabelsManager extends AbstractService {
conf.getClass(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS,
FileSystemNodeLabelsStore.class, NodeLabelsStore.class),
conf);
- this.store.setNodeLabelsManager(this);
- this.store.init(conf);
+ this.store.init(conf, this);
this.store.recover();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
index 0ec4ea4..e11e6f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
@@ -18,275 +18,89 @@
package org.apache.hadoop.yarn.nodelabels;
-import java.io.EOFException;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocalFileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto;
-import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.RemoveFromClusterNodeLabelsRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl;
+import org.apache.hadoop.yarn.nodelabels.store.AbstractFSNodeStore;
+import org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler;
-import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.nodelabels.store.op.AddClusterLabelOp;
+import org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler.StoreType;
+import org.apache.hadoop.yarn.nodelabels.store.op.NodeToLabelOp;
+import org.apache.hadoop.yarn.nodelabels.store.op.RemoveClusterLabelOp;
-public class FileSystemNodeLabelsStore extends NodeLabelsStore {
- protected static final Log LOG = LogFactory.getLog(FileSystemNodeLabelsStore.class);
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class FileSystemNodeLabelsStore
+ extends AbstractFSNodeStore<CommonNodeLabelsManager>
+ implements NodeLabelsStore {
+ protected static final Log LOG =
+ LogFactory.getLog(FileSystemNodeLabelsStore.class);
protected static final String DEFAULT_DIR_NAME = "node-labels";
protected static final String MIRROR_FILENAME = "nodelabel.mirror";
protected static final String EDITLOG_FILENAME = "nodelabel.editlog";
-
- protected enum SerializedLogType {
- ADD_LABELS, NODE_TO_LABELS, REMOVE_LABELS
+
+ FileSystemNodeLabelsStore() {
+ super(StoreType.NODE_LABEL_STORE);
}
- Path fsWorkingPath;
- FileSystem fs;
- private FSDataOutputStream editlogOs;
- private Path editLogPath;
-
private String getDefaultFSNodeLabelsRootDir() throws IOException {
// default is in local: /tmp/hadoop-yarn-${user}/node-labels/
- return "file:///tmp/hadoop-yarn-"
- + UserGroupInformation.getCurrentUser().getShortUserName() + "/"
- + DEFAULT_DIR_NAME;
+ return "file:///tmp/hadoop-yarn-" + UserGroupInformation.getCurrentUser()
+ .getShortUserName() + "/" + DEFAULT_DIR_NAME;
}
@Override
- public void init(Configuration conf) throws Exception {
- fsWorkingPath =
- new Path(conf.get(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
- getDefaultFSNodeLabelsRootDir()));
-
- setFileSystem(conf);
-
- // mkdir of root dir path
- fs.mkdirs(fsWorkingPath);
+ public void init(Configuration conf, CommonNodeLabelsManager mgr)
+ throws Exception {
+ StoreSchema schema = new StoreSchema(EDITLOG_FILENAME, MIRROR_FILENAME);
+ initStore(conf, new Path(
+ conf.get(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+ getDefaultFSNodeLabelsRootDir())), schema, mgr);
}
@Override
public void close() throws IOException {
- IOUtils.cleanup(LOG, fs, editlogOs);
- }
-
- void setFileSystem(Configuration conf) throws IOException {
- Configuration confCopy = new Configuration(conf);
- fs = fsWorkingPath.getFileSystem(confCopy);
-
- // if it's local file system, use RawLocalFileSystem instead of
- // LocalFileSystem, the latter one doesn't support append.
- if (fs.getScheme().equals("file")) {
- fs = ((LocalFileSystem)fs).getRaw();
- }
- }
-
- private void ensureAppendEditlogFile() throws IOException {
- editlogOs = fs.append(editLogPath);
- }
-
- private void ensureCloseEditlogFile() throws IOException {
- editlogOs.close();
+ super.closeFSStore();
}
@Override
- public void updateNodeToLabelsMappings(
- Map<NodeId, Set<String>> nodeToLabels) throws IOException {
- try {
- ensureAppendEditlogFile();
- editlogOs.writeInt(SerializedLogType.NODE_TO_LABELS.ordinal());
- ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest
- .newInstance(nodeToLabels)).getProto().writeDelimitedTo(editlogOs);
- } finally {
- ensureCloseEditlogFile();
- }
+ public void updateNodeToLabelsMappings(Map<NodeId, Set<String>> nodeToLabels)
+ throws IOException {
+ NodeToLabelOp op = new NodeToLabelOp();
+ writeToLog(op.setNodeToLabels(nodeToLabels));
}
@Override
public void storeNewClusterNodeLabels(List<NodeLabel> labels)
throws IOException {
- try {
- ensureAppendEditlogFile();
- editlogOs.writeInt(SerializedLogType.ADD_LABELS.ordinal());
- ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest
- .newInstance(labels)).getProto().writeDelimitedTo(editlogOs);
- } finally {
- ensureCloseEditlogFile();
- }
+ AddClusterLabelOp op = new AddClusterLabelOp();
+ writeToLog(op.setLabels(labels));
}
@Override
public void removeClusterNodeLabels(Collection<String> labels)
throws IOException {
- try {
- ensureAppendEditlogFile();
- editlogOs.writeInt(SerializedLogType.REMOVE_LABELS.ordinal());
- ((RemoveFromClusterNodeLabelsRequestPBImpl) RemoveFromClusterNodeLabelsRequest.newInstance(Sets
- .newHashSet(labels.iterator()))).getProto().writeDelimitedTo(editlogOs);
- } finally {
- ensureCloseEditlogFile();
- }
- }
-
- protected void loadFromMirror(Path newMirrorPath, Path oldMirrorPath)
- throws IOException {
- // If mirror.new exists, read from mirror.new,
- FSDataInputStream is = null;
- try {
- is = fs.open(newMirrorPath);
- } catch (FileNotFoundException e) {
- try {
- is = fs.open(oldMirrorPath);
- } catch (FileNotFoundException ignored) {
-
- }
- }
- if (null != is) {
- List<NodeLabel> labels = new AddToClusterNodeLabelsRequestPBImpl(
- AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is))
- .getNodeLabels();
- mgr.addToCluserNodeLabels(labels);
-
- if (mgr.isCentralizedConfiguration()) {
- // Only load node to labels mapping while using centralized configuration
- Map<NodeId, Set<String>> nodeToLabels =
- new ReplaceLabelsOnNodeRequestPBImpl(
- ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
- .getNodeToLabels();
- mgr.replaceLabelsOnNode(nodeToLabels);
- }
- is.close();
- }
+ RemoveClusterLabelOp op = new RemoveClusterLabelOp();
+ writeToLog(op.setLabels(labels));
}
/* (non-Javadoc)
- * @see org.apache.hadoop.yarn.nodelabels.NodeLabelsStore#recover(boolean)
- */
- @Override
- public void recover() throws YarnException,
- IOException {
- /*
- * Steps of recover
- * 1) Read from last mirror (from mirror or mirror.old)
- * 2) Read from last edit log, and apply such edit log
- * 3) Write new mirror to mirror.writing
- * 4) Rename mirror to mirror.old
- * 5) Move mirror.writing to mirror
- * 6) Remove mirror.old
- * 7) Remove edit log and create a new empty edit log
+ * @see org.apache.hadoop.yarn.nodelabels.NodeLabelsStore#recover(boolean)
*/
-
- // Open mirror from serialized file
- Path mirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
- Path oldMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".old");
-
- loadFromMirror(mirrorPath, oldMirrorPath);
-
- // Open and process editlog
- editLogPath = new Path(fsWorkingPath, EDITLOG_FILENAME);
- FSDataInputStream is;
- try {
- is = fs.open(editLogPath);
- } catch (FileNotFoundException e) {
- is = null;
- }
- if (null != is) {
-
- while (true) {
- try {
- // read edit log one by one
- SerializedLogType type = SerializedLogType.values()[is.readInt()];
-
- switch (type) {
- case ADD_LABELS: {
- List<NodeLabel> labels =
- new AddToClusterNodeLabelsRequestPBImpl(
- AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is))
- .getNodeLabels();
- mgr.addToCluserNodeLabels(labels);
- break;
- }
- case REMOVE_LABELS: {
- Collection<String> labels =
- RemoveFromClusterNodeLabelsRequestProto.parseDelimitedFrom(is)
- .getNodeLabelsList();
- mgr.removeFromClusterNodeLabels(labels);
- break;
- }
- case NODE_TO_LABELS: {
- Map<NodeId, Set<String>> map =
- new ReplaceLabelsOnNodeRequestPBImpl(
- ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
- .getNodeToLabels();
- if (mgr.isCentralizedConfiguration()) {
- /*
- * In case of Distributed NodeLabels setup,
- * ignoreNodeToLabelsMappings will be set to true and recover will
- * be invoked. As RM will collect the node labels from NM through
- * registration/HB
- */
- mgr.replaceLabelsOnNode(map);
- }
- break;
- }
- }
- } catch (EOFException e) {
- // EOF hit, break
- break;
- }
- }
- is.close();
- }
-
- // Serialize current mirror to mirror.writing
- Path writingMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".writing");
- FSDataOutputStream os = fs.create(writingMirrorPath, true);
- ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequestPBImpl
- .newInstance(mgr.getClusterNodeLabels())).getProto().writeDelimitedTo(os);
- ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest
- .newInstance(mgr.getNodeLabels())).getProto().writeDelimitedTo(os);
- os.close();
-
- // Move mirror to mirror.old
- if (fs.exists(mirrorPath)) {
- fs.delete(oldMirrorPath, false);
- fs.rename(mirrorPath, oldMirrorPath);
- }
-
- // move mirror.writing to mirror
- fs.rename(writingMirrorPath, mirrorPath);
- fs.delete(writingMirrorPath, false);
-
- // remove mirror.old
- fs.delete(oldMirrorPath, false);
-
- // create a new editlog file
- editlogOs = fs.create(editLogPath, true);
- editlogOs.close();
-
- LOG.info("Finished write mirror at:" + mirrorPath.toString());
- LOG.info("Finished create editlog file at:" + editLogPath.toString());
+ @Override
+ public void recover() throws YarnException, IOException {
+ super.recoverFromStore();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
index aacb920..e4efd68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
@@ -30,25 +30,27 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.exceptions.YarnException;
-public abstract class NodeLabelsStore implements Closeable {
- protected CommonNodeLabelsManager mgr;
-
+/**
+ * Interface class for Node label store.
+ */
+public interface NodeLabelsStore extends Closeable {
+
/**
- * Store node {@literal ->} label
+ * Store node {@literal ->} label.
*/
- public abstract void updateNodeToLabelsMappings(
+ void updateNodeToLabelsMappings(
Map<NodeId, Set<String>> nodeToLabels) throws IOException;
/**
- * Store new labels
+ * Store new labels.
*/
- public abstract void storeNewClusterNodeLabels(List<NodeLabel> label)
+ void storeNewClusterNodeLabels(List<NodeLabel> label)
throws IOException;
/**
- * Remove labels
+ * Remove labels.
*/
- public abstract void removeClusterNodeLabels(Collection<String> labels)
+ void removeClusterNodeLabels(Collection<String> labels)
throws IOException;
/**
@@ -56,16 +58,14 @@ public abstract class NodeLabelsStore implements Closeable {
* ignoreNodeToLabelsMappings is true then node to labels mappings should not
* be recovered. In case of Distributed NodeLabels setup
* ignoreNodeToLabelsMappings will be set to true and recover will be invoked
- * as RM will collect the node labels from NM through registration/HB
+ * as RM will collect the node labels from NM through registration/HB.
*
* @throws IOException
* @throws YarnException
*/
- public abstract void recover() throws IOException, YarnException;
-
- public void init(Configuration conf) throws Exception {}
+ void recover() throws IOException, YarnException;
+
+ void init(Configuration conf, CommonNodeLabelsManager mgr)
+ throws Exception;
- public void setNodeLabelsManager(CommonNodeLabelsManager mgr) {
- this.mgr = mgr;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
index 989f027..29bfff9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
@@ -18,13 +18,6 @@
package org.apache.hadoop.yarn.nodelabels;
-import java.io.IOException;
-import java.util.Collection;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FSDataOutputStream;
@@ -32,11 +25,19 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl;
+import org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler;
+import org.apache.hadoop.yarn.nodelabels.store.StoreOp;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+/**
+ * Store implementation for Non Appendable File Store
+ */
public class NonAppendableFSNodeLabelStore extends FileSystemNodeLabelsStore {
protected static final Log
LOG = LogFactory.getLog(NonAppendableFSNodeLabelStore.class);
@@ -52,7 +53,7 @@ public class NonAppendableFSNodeLabelStore extends FileSystemNodeLabelsStore {
Path newMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new");
Path oldMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
loadFromMirror(newMirrorPath, oldMirrorPath);
-
+
// if new mirror exists, remove old mirror and rename new mirror
if (fs.exists(newMirrorPath)) {
// remove old mirror
@@ -91,29 +92,18 @@ public class NonAppendableFSNodeLabelStore extends FileSystemNodeLabelsStore {
}
private void writeNewMirror() throws IOException {
- ReentrantReadWriteLock.ReadLock readLock = mgr.readLock;
+ ReentrantReadWriteLock.ReadLock readLock = manager.readLock;
try {
// Acquire readlock to make sure we get cluster node labels and
// node-to-labels mapping atomically.
readLock.lock();
- List<NodeLabel> nodeLabels = mgr.getClusterNodeLabels();
- Map<NodeId, Set<String>> nodeToLabels = mgr.getNodeLabels();
-
// Write mirror to mirror.new.tmp file
- Path newTmpPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new.tmp");
- FSDataOutputStream os = fs
- .create(newTmpPath, true);
- ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest
- .newInstance(nodeLabels)).getProto().writeDelimitedTo(os);
-
- if (mgr.isCentralizedConfiguration()) {
- // Only save node-to-labels mapping while using centralized configuration
- ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest
- .newInstance(nodeToLabels)).getProto().writeDelimitedTo(os);
+ Path newTmpPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new.tmp");
+ try (FSDataOutputStream os = fs.create(newTmpPath, true)) {
+ StoreOp op = FSStoreOpHandler.getMirrorOp(getStoreType());
+ op.write(os, manager);
}
- os.close();
-
// Rename mirror.new.tmp to mirror.new (will remove .new if it's existed)
Path newPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new");
fs.delete(newPath, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
new file mode 100644
index 0000000..a47cacf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/AbstractFSNodeStore.java
@@ -0,0 +1,213 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.yarn.nodelabels.store.op.FSNodeStoreLogOp;
+import org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler.StoreType;
+
+import java.io.EOFException;
+import java.io.IOException;
+
+/**
+ * Abstract class for File System based store.
+ *
+ * @param <M> manager filesystem store.Currently nodelabel will use
+ * CommonNodeLabelManager.
+ */
+public abstract class AbstractFSNodeStore<M> {
+
+ protected static final Log LOG = LogFactory.getLog(AbstractFSNodeStore.class);
+
+ private StoreType storeType;
+ private FSDataOutputStream editlogOs;
+
+ private Path editLogPath;
+ private StoreSchema schema;
+
+ protected M manager;
+ protected Path fsWorkingPath;
+ protected FileSystem fs;
+
+ public AbstractFSNodeStore(StoreType storeType) {
+ this.storeType = storeType;
+ }
+
+ protected void initStore(Configuration conf, Path fsStorePath,
+ StoreSchema schma, M mgr) throws IOException {
+ this.schema = schma;
+ this.fsWorkingPath = fsStorePath;
+ this.manager = mgr;
+ initFileSystem(conf);
+ // mkdir of root dir path
+ fs.mkdirs(fsWorkingPath);
+
+ }
+
+ /**
+ * Filesystem store schema define the log name and mirror name.
+ */
+ public static class StoreSchema {
+ private String editLogName;
+ private String mirrorName;
+
+ public StoreSchema(String editLogName, String mirrorName) {
+ this.editLogName = editLogName;
+ this.mirrorName = mirrorName;
+ }
+ }
+
+ public void initFileSystem(Configuration conf) throws IOException {
+ Configuration confCopy = new Configuration(conf);
+ fs = fsWorkingPath.getFileSystem(confCopy);
+ // if it's local file system, use RawLocalFileSystem instead of
+ // LocalFileSystem, the latter one doesn't support append.
+ if (fs.getScheme().equals("file")) {
+ fs = ((LocalFileSystem) fs).getRaw();
+ }
+ }
+
+ protected void writeToLog(FSNodeStoreLogOp op) throws IOException {
+ try {
+ ensureAppendEditLogFile();
+ editlogOs.writeInt(op.getOpCode());
+ op.write(editlogOs, manager);
+ } finally {
+ ensureCloseEditlogFile();
+ }
+ }
+
+ protected void ensureAppendEditLogFile() throws IOException {
+ editlogOs = fs.append(editLogPath);
+ }
+
+ protected void ensureCloseEditlogFile() throws IOException {
+ editlogOs.close();
+ }
+
+ protected void loadFromMirror(Path newMirrorPath, Path oldMirrorPath)
+ throws IOException {
+ // If mirror.new exists, read from mirror.new
+ Path mirrorToRead = fs.exists(newMirrorPath) ?
+ newMirrorPath :
+ fs.exists(oldMirrorPath) ? oldMirrorPath : null;
+ if (mirrorToRead != null) {
+ try (FSDataInputStream is = fs.open(mirrorToRead)) {
+ StoreOp op = FSStoreOpHandler.getMirrorOp(storeType);
+ op.recover(is, manager);
+ }
+ }
+ }
+
+ protected StoreType getStoreType() {
+ return storeType;
+ }
+
+ public Path getFsWorkingPath() {
+ return fsWorkingPath;
+ }
+
+ protected void recoverFromStore() throws IOException {
+ /*
+ * Steps of recover
+ * 1) Read from last mirror (from mirror or mirror.old)
+ * 2) Read from last edit log, and apply such edit log
+ * 3) Write new mirror to mirror.writing
+ * 4) Rename mirror to mirror.old
+ * 5) Move mirror.writing to mirror
+ * 6) Remove mirror.old
+ * 7) Remove edit log and create a new empty edit log
+ */
+
+ // Open mirror from serialized file
+ Path mirrorPath = new Path(fsWorkingPath, schema.mirrorName);
+ Path oldMirrorPath = new Path(fsWorkingPath, schema.mirrorName + ".old");
+
+ loadFromMirror(mirrorPath, oldMirrorPath);
+
+ // Open and process editlog
+ editLogPath = new Path(fsWorkingPath, schema.editLogName);
+
+ loadManagerFromEditLog(editLogPath);
+
+ // Serialize current mirror to mirror.writing
+ Path writingMirrorPath =
+ new Path(fsWorkingPath, schema.mirrorName + ".writing");
+
+ try(FSDataOutputStream os = fs.create(writingMirrorPath, true)){
+ StoreOp op = FSStoreOpHandler.getMirrorOp(storeType);
+ op.write(os, manager);
+ }
+ // Move mirror to mirror.old
+ if (fs.exists(mirrorPath)) {
+ fs.delete(oldMirrorPath, false);
+ fs.rename(mirrorPath, oldMirrorPath);
+ }
+
+ // move mirror.writing to mirror
+ fs.rename(writingMirrorPath, mirrorPath);
+ fs.delete(writingMirrorPath, false);
+
+ // remove mirror.old
+ fs.delete(oldMirrorPath, false);
+
+ // create a new editlog file
+ editlogOs = fs.create(editLogPath, true);
+ editlogOs.close();
+
+ LOG.info("Finished write mirror at:" + mirrorPath.toString());
+ LOG.info("Finished create editlog file at:" + editLogPath.toString());
+ }
+
+ protected void loadManagerFromEditLog(Path editLogPath) throws IOException {
+ if (!fs.exists(editLogPath)) {
+ return;
+ }
+ try (FSDataInputStream is = fs.open(editLogPath)) {
+ while (true) {
+ try {
+ StoreOp storeOp = FSStoreOpHandler.get(is.readInt(),storeType);
+ storeOp.recover(is, manager);
+ } catch (EOFException e) {
+ // EOF hit, break
+ break;
+ }
+ }
+ }
+ }
+
+ public FileSystem getFs() {
+ return fs;
+ }
+
+ public void setFs(FileSystem fs) {
+ this.fs = fs;
+ }
+
+ protected void closeFSStore() {
+ IOUtils.closeStreams(fs, editlogOs);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java
new file mode 100644
index 0000000..0f7f53d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/FSStoreOpHandler.java
@@ -0,0 +1,108 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store;
+
+import static org.apache.hadoop.yarn.nodelabels.store.FSStoreOpHandler
+ .StoreType.NODE_LABEL_STORE;
+import org.apache.hadoop.yarn.nodelabels.store.op.AddClusterLabelOp;
+import org.apache.hadoop.yarn.nodelabels.store.op.FSNodeStoreLogOp;
+import org.apache.hadoop.yarn.nodelabels.store.op.NodeLabelMirrorOp;
+import org.apache.hadoop.yarn.nodelabels.store.op.NodeToLabelOp;
+import org.apache.hadoop.yarn.nodelabels.store.op.RemoveClusterLabelOp;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * File system store op handler.
+ */
+public class FSStoreOpHandler {
+
+ private static Map<StoreType, Map<Integer, Class<? extends FSNodeStoreLogOp>>>
+ editLogOp;
+ private static Map<StoreType, Class<? extends FSNodeStoreLogOp>> mirrorOp;
+
+ public enum StoreType {
+ NODE_LABEL_STORE,
+ NODE_LABEL_ATTRIBUTE;
+ }
+
+ static {
+ editLogOp = new HashMap<>();
+ mirrorOp = new HashMap<>();
+
+ // registerLog edit log operation
+ registerLog(NODE_LABEL_STORE, AddClusterLabelOp.OPCODE, AddClusterLabelOp.class);
+ registerLog(NODE_LABEL_STORE, NodeToLabelOp.OPCODE, NodeToLabelOp.class);
+ registerLog(NODE_LABEL_STORE, RemoveClusterLabelOp.OPCODE, RemoveClusterLabelOp.class);
+
+ // registerLog Mirror op
+
+ registerMirror(NODE_LABEL_STORE, NodeLabelMirrorOp.class);
+ }
+
+ private static void registerMirror(StoreType type,
+ Class<? extends FSNodeStoreLogOp> clazz) {
+ mirrorOp.put(type, clazz);
+ }
+
+ private static void registerLog(StoreType type, int opcode,
+ Class<? extends FSNodeStoreLogOp> clazz) {
+ Map<Integer, Class<? extends FSNodeStoreLogOp>> ops = editLogOp.get(type);
+ Integer code = Integer.valueOf(opcode);
+ if (ops == null) {
+ Map<Integer, Class<? extends FSNodeStoreLogOp>> newOps = new HashMap<>();
+ newOps.put(code, clazz);
+ editLogOp.put(type, newOps);
+ } else {
+ ops.put(code, clazz);
+ }
+ }
+
+ /**
+ * Get mirror operation of store Type.
+ *
+ * @param storeType
+ * @return instance of FSNodeStoreLogOp.
+ */
+ public static FSNodeStoreLogOp getMirrorOp(StoreType storeType) {
+ return newInstance(mirrorOp.get(storeType));
+ }
+
+ /**
+ * Will return StoreOp instance basead on opCode and StoreType.
+ * @param opCode
+ * @param storeType
+ * @return instance of FSNodeStoreLogOp.
+ */
+ public static FSNodeStoreLogOp get(int opCode, StoreType storeType) {
+ return newInstance(editLogOp.get(storeType).get(opCode));
+ }
+
+ private static <T extends FSNodeStoreLogOp> T newInstance(Class<T> clazz) {
+ FSNodeStoreLogOp instance = null;
+ if (clazz != null) {
+ try {
+ instance = clazz.newInstance();
+ } catch (Exception ex) {
+ throw new RuntimeException("Failed to instantiate " + clazz, ex);
+ }
+ }
+ return (T) instance;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java
new file mode 100644
index 0000000..c26e1dc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/StoreOp.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store;
+
+import java.io.IOException;
+
+/**
+ * Define the interface for store activity.
+ * Used by for FileSystem based operation.
+ *
+ * @param <W> write to be done to
+ * @param <R> read to be done from
+ * @param <M> manager used
+ */
+public interface StoreOp<W, R, M> {
+
+ /**
+ * Write operation to persistent storage
+ *
+ * @param write write to be done to
+ * @param mgr manager used by store
+ * @throws IOException
+ */
+ void write(W write, M mgr) throws IOException;
+
+ /**
+ * Read and populate StoreOp
+ *
+ * @param read read to be done from
+ * @param mgr manager used by store
+ * @throws IOException
+ */
+ void recover(R read, M mgr) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java
new file mode 100644
index 0000000..ce736aa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/AddClusterLabelOp.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store.op;
+
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
+import org.apache.hadoop.yarn.server.api.protocolrecords
+ .AddToClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .AddToClusterNodeLabelsRequestPBImpl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+
+/**
+ * Add label operation for file system.
+ */
+public class AddClusterLabelOp
+ extends FSNodeStoreLogOp<CommonNodeLabelsManager> {
+
+ private List<NodeLabel> labels;
+
+ public static final int OPCODE = 0;
+
+ @Override
+ public void write(OutputStream os, CommonNodeLabelsManager mgr)
+ throws IOException {
+ ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest
+ .newInstance(labels)).getProto().writeDelimitedTo(os);
+ }
+
+ @Override
+ public void recover(InputStream is, CommonNodeLabelsManager mgr)
+ throws IOException {
+ labels = new AddToClusterNodeLabelsRequestPBImpl(
+ YarnServerResourceManagerServiceProtos
+ .AddToClusterNodeLabelsRequestProto
+ .parseDelimitedFrom(is)).getNodeLabels();
+ mgr.addToCluserNodeLabels(labels);
+ }
+
+ public AddClusterLabelOp setLabels(List<NodeLabel> labels) {
+ this.labels = labels;
+ return this;
+ }
+
+ public List<NodeLabel> getLabels() {
+ return labels;
+ }
+
+ @Override
+ public int getOpCode() {
+ return OPCODE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/FSNodeStoreLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/FSNodeStoreLogOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/FSNodeStoreLogOp.java
new file mode 100644
index 0000000..cd739c0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/FSNodeStoreLogOp.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store.op;
+
+import org.apache.hadoop.yarn.nodelabels.store.StoreOp;
+
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/**
+ * Defines all FileSystem editlog operation. All node label and attribute
+ * store write or read operation will be defined in this class.
+ *
+ * @param <M> Manager used for each operation.
+ */
+public abstract class FSNodeStoreLogOp<M>
+ implements StoreOp<OutputStream, InputStream, M> {
+
+ public abstract int getOpCode();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java
new file mode 100644
index 0000000..3ec837b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeLabelMirrorOp.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store.op;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
+import org.apache.hadoop.yarn.server.api.protocolrecords
+ .ReplaceLabelsOnNodeRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .AddToClusterNodeLabelsRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .ReplaceLabelsOnNodeRequestPBImpl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class NodeLabelMirrorOp
+ extends FSNodeStoreLogOp<CommonNodeLabelsManager> {
+
+ public NodeLabelMirrorOp() {
+ super();
+ }
+
+ @Override
+ public void write(OutputStream os, CommonNodeLabelsManager mgr)
+ throws IOException {
+ ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequestPBImpl
+ .newInstance(mgr.getClusterNodeLabels())).getProto()
+ .writeDelimitedTo(os);
+ if (mgr.isCentralizedConfiguration()) {
+ ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest
+ .newInstance(mgr.getNodeLabels())).getProto().writeDelimitedTo(os);
+ }
+ }
+
+ @Override
+ public void recover(InputStream is, CommonNodeLabelsManager mgr)
+ throws IOException {
+ List<NodeLabel> labels = new AddToClusterNodeLabelsRequestPBImpl(
+ YarnServerResourceManagerServiceProtos
+ .AddToClusterNodeLabelsRequestProto
+ .parseDelimitedFrom(is)).getNodeLabels();
+ mgr.addToCluserNodeLabels(labels);
+
+ if (mgr.isCentralizedConfiguration()) {
+ // Only load node to labels mapping while using centralized
+ // configuration
+ Map<NodeId, Set<String>> nodeToLabels =
+ new ReplaceLabelsOnNodeRequestPBImpl(
+ YarnServerResourceManagerServiceProtos
+ .ReplaceLabelsOnNodeRequestProto
+ .parseDelimitedFrom(is)).getNodeToLabels();
+ mgr.replaceLabelsOnNode(nodeToLabels);
+ }
+ }
+
+ @Override
+ public int getOpCode() {
+ return -1;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java
new file mode 100644
index 0000000..0e1e398
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/NodeToLabelOp.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store.op;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
+import org.apache.hadoop.yarn.server.api.protocolrecords
+ .ReplaceLabelsOnNodeRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .ReplaceLabelsOnNodeRequestPBImpl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Node to label mapping store operation for label.
+ */
+public class NodeToLabelOp
+ extends FSNodeStoreLogOp<CommonNodeLabelsManager> {
+
+ private Map<NodeId, Set<String>> nodeToLabels;
+ public static final int OPCODE = 1;
+
+ @Override
+ public void write(OutputStream os, CommonNodeLabelsManager mgr)
+ throws IOException {
+ ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest
+ .newInstance(nodeToLabels)).getProto().writeDelimitedTo(os);
+ }
+
+ @Override
+ public void recover(InputStream is, CommonNodeLabelsManager mgr)
+ throws IOException {
+ nodeToLabels = new ReplaceLabelsOnNodeRequestPBImpl(
+ YarnServerResourceManagerServiceProtos.ReplaceLabelsOnNodeRequestProto
+ .parseDelimitedFrom(is)).getNodeToLabels();
+ if (mgr.isCentralizedConfiguration()) {
+ mgr.replaceLabelsOnNode(nodeToLabels);
+ }
+ }
+
+ public NodeToLabelOp setNodeToLabels(
+ Map<NodeId, Set<String>> nodeToLabels) {
+ this.nodeToLabels = nodeToLabels;
+ return this;
+ }
+
+ public Map<NodeId, Set<String>> getNodeToLabels() {
+ return nodeToLabels;
+ }
+
+ @Override
+ public int getOpCode() {
+ return OPCODE;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java
new file mode 100644
index 0000000..4f6d4bd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/op/RemoveClusterLabelOp.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels.store.op;
+
+import com.google.common.collect.Sets;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos;
+import org.apache.hadoop.yarn.server.api.protocolrecords
+ .RemoveFromClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb
+ .RemoveFromClusterNodeLabelsRequestPBImpl;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.Collection;
+
+/**
+ * Remove label from cluster log store operation.
+ */
+public class RemoveClusterLabelOp
+ extends FSNodeStoreLogOp<CommonNodeLabelsManager> {
+
+ private Collection<String> labels;
+
+ public static final int OPCODE = 2;
+
+ @Override
+ public void write(OutputStream os, CommonNodeLabelsManager mgr)
+ throws IOException {
+ ((RemoveFromClusterNodeLabelsRequestPBImpl)
+ RemoveFromClusterNodeLabelsRequest
+ .newInstance(Sets.newHashSet(labels.iterator()))).getProto()
+ .writeDelimitedTo(os);
+ }
+
+ @Override
+ public void recover(InputStream is, CommonNodeLabelsManager mgr)
+ throws IOException {
+ labels =
+ YarnServerResourceManagerServiceProtos
+ .RemoveFromClusterNodeLabelsRequestProto
+ .parseDelimitedFrom(is).getNodeLabelsList();
+ mgr.removeFromClusterNodeLabels(labels);
+ }
+
+ public RemoveClusterLabelOp setLabels(Collection<String> labels) {
+ this.labels = labels;
+ return this;
+ }
+
+ public Collection<String> getLabels() {
+ return labels;
+ }
+
+ @Override
+ public int getOpCode() {
+ return OPCODE;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java
new file mode 100644
index 0000000..0444807
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/store/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+package org.apache.hadoop.yarn.nodelabels.store;
+import org.apache.hadoop.classification.InterfaceAudience;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java
index a935840..f633804 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/RemoveFromClusterNodeLabelsRequestPBImpl.java
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -21,12 +21,15 @@ package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
import java.util.HashSet;
import java.util.Set;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.RemoveFromClusterNodeLabelsRequestProto;
-import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos
+ .RemoveFromClusterNodeLabelsRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos
+ .RemoveFromClusterNodeLabelsRequestProto;
+import org.apache.hadoop.yarn.server.api.protocolrecords
+ .RemoveFromClusterNodeLabelsRequest;
-public class RemoveFromClusterNodeLabelsRequestPBImpl extends
- RemoveFromClusterNodeLabelsRequest {
+public class RemoveFromClusterNodeLabelsRequestPBImpl
+ extends RemoveFromClusterNodeLabelsRequest {
Set<String> labels;
RemoveFromClusterNodeLabelsRequestProto proto =
RemoveFromClusterNodeLabelsRequestProto.getDefaultInstance();
@@ -102,7 +105,7 @@ public class RemoveFromClusterNodeLabelsRequestPBImpl extends
assert false : "hashCode not designed";
return 0;
}
-
+
@Override
public boolean equals(Object other) {
if (other == null)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
index 64c74c2..61373dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
@@ -44,6 +44,12 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager {
}
@Override
+ public void init(Configuration conf, CommonNodeLabelsManager mgr)
+ throws Exception {
+
+ }
+
+ @Override
public void removeClusterNodeLabels(Collection<String> labels)
throws IOException {
lastRemovedlabels = labels;
@@ -65,8 +71,6 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager {
// do nothing
}
};
-
- this.store.setNodeLabelsManager(this);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index ed2f4aa..93c039a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -96,7 +96,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
if (mgr.store instanceof FileSystemNodeLabelsStore) {
FileSystemNodeLabelsStore fsStore =
((FileSystemNodeLabelsStore) mgr.store);
- fsStore.fs.delete(fsStore.fsWorkingPath, true);
+ fsStore.getFs().delete(fsStore.getFsWorkingPath(), true);
}
mgr.stop();
}
@@ -342,12 +342,12 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
public void testRootMkdirOnInitStore() throws Exception {
final FileSystem mockFs = Mockito.mock(FileSystem.class);
FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore() {
- void setFileSystem(Configuration conf) throws IOException {
- fs = mockFs;
+ public void initFileSystem(Configuration config) throws IOException {
+ setFs(mockFs);
}
};
- mockStore.setNodeLabelsManager(mgr);
- mockStore.fs = mockFs;
+
+ mockStore.setFs(mockFs);
verifyMkdirsCount(mockStore, true, 1);
verifyMkdirsCount(mockStore, false, 2);
verifyMkdirsCount(mockStore, true, 3);
@@ -357,10 +357,10 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
private void verifyMkdirsCount(FileSystemNodeLabelsStore store,
boolean existsRetVal, int expectedNumOfCalls)
throws Exception {
- Mockito.when(store.fs.exists(Mockito.any(
+ Mockito.when(store.getFs().exists(Mockito.any(
Path.class))).thenReturn(existsRetVal);
- store.init(conf);
- Mockito.verify(store.fs,Mockito.times(
+ store.init(conf, mgr);
+ Mockito.verify(store.getFs(),Mockito.times(
expectedNumOfCalls)).mkdirs(Mockito.any(Path
.class));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
index bb0b45f..b8f3fae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelsStore;
public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
@@ -46,6 +47,12 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
}
@Override
+ public void init(Configuration conf, CommonNodeLabelsManager mgr)
+ throws Exception {
+
+ }
+
+ @Override
public void removeClusterNodeLabels(Collection<String> labels)
throws IOException {
// do nothing
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a247908/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index 5e6fd4e..21df698 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -605,7 +605,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTestBase {
String expectedmessage =
"java.io.IOException: label name should only contains"
+ " {0-9, a-z, A-Z, -, _} and should not started with"
- + " {-,_}, now it is=a&";
+ + " {-,_}, now it is= a&";
validateJsonExceptionContent(response, expectedmessage);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: HDFS-13347. RBF: Cache datanode
reports. Contributed by Inigo Goiri.
Posted by na...@apache.org.
HDFS-13347. RBF: Cache datanode reports. Contributed by Inigo Goiri.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a71656c1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a71656c1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a71656c1
Branch: refs/heads/YARN-3409
Commit: a71656c1c1bf6c680f1382a76ddcac870061f320
Parents: d1e378d
Author: Yiqun Lin <yq...@apache.org>
Authored: Wed Mar 28 11:00:08 2018 +0800
Committer: Yiqun Lin <yq...@apache.org>
Committed: Wed Mar 28 11:00:08 2018 +0800
----------------------------------------------------------------------
.../federation/metrics/FederationMetrics.java | 2 +-
.../federation/metrics/NamenodeBeanMetrics.java | 68 +++++++++++++++++++-
.../hdfs/server/federation/router/Router.java | 13 ++++
.../federation/router/RouterMetricsService.java | 9 +++
.../federation/router/RouterRpcServer.java | 16 ++---
.../server/federation/router/TestRouterRpc.java | 41 +++++++++++-
6 files changed, 135 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
index a80c3be..a99a26a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/FederationMetrics.java
@@ -434,7 +434,7 @@ public class FederationMetrics implements FederationMBean {
try {
RouterRpcServer rpcServer = this.router.getRpcServer();
DatanodeInfo[] live = rpcServer.getDatanodeReport(
- DatanodeReportType.LIVE, TIME_OUT);
+ DatanodeReportType.LIVE, false, TIME_OUT);
if (live.length > 0) {
float totalDfsUsed = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
index c4e5b5b..e8c6c82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/metrics/NamenodeBeanMetrics.java
@@ -25,6 +25,8 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -32,6 +34,7 @@ import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
import javax.management.StandardMBean;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -39,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.federation.resolver.FederationNamespaceInfo;
+import org.apache.hadoop.hdfs.server.federation.router.RBFConfigKeys;
import org.apache.hadoop.hdfs.server.federation.router.Router;
import org.apache.hadoop.hdfs.server.federation.router.RouterRpcServer;
import org.apache.hadoop.hdfs.server.federation.store.MembershipStore;
@@ -56,6 +60,10 @@ import org.eclipse.jetty.util.ajax.JSON;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+
/**
* Expose the Namenode metrics as the Router was one.
*/
@@ -65,6 +73,22 @@ public class NamenodeBeanMetrics
private static final Logger LOG =
LoggerFactory.getLogger(NamenodeBeanMetrics.class);
+ /** Prevent holding the page from loading too long. */
+ private static final String DN_REPORT_TIME_OUT =
+ RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.time-out";
+ /** We only wait for 1 second. */
+ private static final long DN_REPORT_TIME_OUT_DEFAULT =
+ TimeUnit.SECONDS.toMillis(1);
+
+ /** Time to cache the DN information. */
+ public static final String DN_REPORT_CACHE_EXPIRE =
+ RBFConfigKeys.FEDERATION_ROUTER_PREFIX + "dn-report.cache-expire";
+ /** We cache the DN information for 10 seconds by default. */
+ public static final long DN_REPORT_CACHE_EXPIRE_DEFAULT =
+ TimeUnit.SECONDS.toMillis(10);
+
+
+ /** Instance of the Router being monitored. */
private final Router router;
/** FSNamesystem bean. */
@@ -76,6 +100,11 @@ public class NamenodeBeanMetrics
/** NameNodeStatus bean. */
private ObjectName nnStatusBeanName;
+ /** Timeout to get the DN report. */
+ private final long dnReportTimeOut;
+ /** DN type -> full DN report in JSON. */
+ private final LoadingCache<DatanodeReportType, String> dnCache;
+
public NamenodeBeanMetrics(Router router) {
this.router = router;
@@ -114,6 +143,23 @@ public class NamenodeBeanMetrics
} catch (NotCompliantMBeanException e) {
throw new RuntimeException("Bad NameNodeStatus MBean setup", e);
}
+
+ // Initialize the cache for the DN reports
+ Configuration conf = router.getConfig();
+ this.dnReportTimeOut = conf.getTimeDuration(
+ DN_REPORT_TIME_OUT, DN_REPORT_TIME_OUT_DEFAULT, TimeUnit.MILLISECONDS);
+ long dnCacheExpire = conf.getTimeDuration(
+ DN_REPORT_CACHE_EXPIRE,
+ DN_REPORT_CACHE_EXPIRE_DEFAULT, TimeUnit.MILLISECONDS);
+ this.dnCache = CacheBuilder.newBuilder()
+ .expireAfterWrite(dnCacheExpire, TimeUnit.MILLISECONDS)
+ .build(
+ new CacheLoader<DatanodeReportType, String>() {
+ @Override
+ public String load(DatanodeReportType type) throws Exception {
+ return getNodesImpl(type);
+ }
+ });
}
/**
@@ -299,16 +345,32 @@ public class NamenodeBeanMetrics
}
/**
+ * Get all the nodes in the federation from a particular type. Getting this
+ * information is expensive and we use a cache.
+ * @param type Type of the datanodes to check.
+ * @return JSON with the nodes.
+ */
+ private String getNodes(final DatanodeReportType type) {
+ try {
+ return this.dnCache.get(type);
+ } catch (ExecutionException e) {
+ LOG.error("Cannot get the DN storage report for {}", type, e);
+ }
+ // If we cannot get the report, return empty JSON
+ return "{}";
+ }
+
+ /**
* Get all the nodes in the federation from a particular type.
- * TODO this is expensive, we may want to cache it.
* @param type Type of the datanodes to check.
* @return JSON with the nodes.
*/
- private String getNodes(DatanodeReportType type) {
+ private String getNodesImpl(final DatanodeReportType type) {
final Map<String, Map<String, Object>> info = new HashMap<>();
try {
RouterRpcServer rpcServer = this.router.getRpcServer();
- DatanodeInfo[] datanodes = rpcServer.getDatanodeReport(type);
+ DatanodeInfo[] datanodes =
+ rpcServer.getDatanodeReport(type, false, dnReportTimeOut);
for (DatanodeInfo node : datanodes) {
Map<String, Object> innerinfo = new HashMap<>();
innerinfo.put("infoAddr", node.getInfoAddr());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
index 38f5d4f..df2a448 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/Router.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.server.federation.metrics.FederationMetrics;
+import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
import org.apache.hadoop.hdfs.server.federation.resolver.ActiveNamenodeResolver;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.federation.store.RouterStore;
@@ -553,6 +554,18 @@ public class Router extends CompositeService {
}
/**
+ * Get the Namenode metrics.
+ *
+ * @return Namenode metrics.
+ */
+ public NamenodeBeanMetrics getNamenodeMetrics() {
+ if (this.metrics != null) {
+ return this.metrics.getNamenodeMetrics();
+ }
+ return null;
+ }
+
+ /**
* Get the subcluster resolver for files.
*
* @return Subcluster resolver for files.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
index f4debce..1887ed6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterMetricsService.java
@@ -95,6 +95,15 @@ public class RouterMetricsService extends AbstractService {
}
/**
+ * Get the Namenode metrics.
+ *
+ * @return Namenode metrics.
+ */
+ public NamenodeBeanMetrics getNamenodeMetrics() {
+ return this.nnMetrics;
+ }
+
+ /**
* Get the JVM metrics for the Router.
*
* @return JVM metrics.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
index eaa3951..383fd77 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java
@@ -1237,18 +1237,20 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
public DatanodeInfo[] getDatanodeReport(DatanodeReportType type)
throws IOException {
checkOperation(OperationCategory.UNCHECKED);
- return getDatanodeReport(type, 0);
+ return getDatanodeReport(type, true, 0);
}
/**
* Get the datanode report with a timeout.
* @param type Type of the datanode.
+ * @param requireResponse If we require all the namespaces to report.
* @param timeOutMs Time out for the reply in milliseconds.
* @return List of datanodes.
* @throws IOException If it cannot get the report.
*/
public DatanodeInfo[] getDatanodeReport(
- DatanodeReportType type, long timeOutMs) throws IOException {
+ DatanodeReportType type, boolean requireResponse, long timeOutMs)
+ throws IOException {
checkOperation(OperationCategory.UNCHECKED);
Map<String, DatanodeInfo> datanodesMap = new LinkedHashMap<>();
@@ -1257,8 +1259,8 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
Set<FederationNamespaceInfo> nss = namenodeResolver.getNamespaces();
Map<FederationNamespaceInfo, DatanodeInfo[]> results =
- rpcClient.invokeConcurrent(
- nss, method, true, false, timeOutMs, DatanodeInfo[].class);
+ rpcClient.invokeConcurrent(nss, method, requireResponse, false,
+ timeOutMs, DatanodeInfo[].class);
for (Entry<FederationNamespaceInfo, DatanodeInfo[]> entry :
results.entrySet()) {
FederationNamespaceInfo ns = entry.getKey();
@@ -1278,9 +1280,7 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
}
// Map -> Array
Collection<DatanodeInfo> datanodes = datanodesMap.values();
- DatanodeInfo[] combinedData = new DatanodeInfo[datanodes.size()];
- combinedData = datanodes.toArray(combinedData);
- return combinedData;
+ return toArray(datanodes, DatanodeInfo.class);
}
@Override // ClientProtocol
@@ -2295,7 +2295,7 @@ public class RouterRpcServer extends AbstractService implements ClientProtocol {
* @param clazz Class of the values.
* @return Array with the values in set.
*/
- private static <T> T[] toArray(Set<T> set, Class<T> clazz) {
+ private static <T> T[] toArray(Collection<T> set, Class<T> clazz) {
@SuppressWarnings("unchecked")
T[] combinedData = (T[]) Array.newInstance(clazz, set.size());
combinedData = set.toArray(combinedData);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a71656c1/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
index e8341a2..5014880 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/test/java/org/apache/hadoop/hdfs/server/federation/router/TestRouterRpc.java
@@ -27,6 +27,7 @@ import static org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.TEST
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -45,6 +46,7 @@ import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
@@ -71,10 +73,11 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
-import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.NamenodeContext;
import org.apache.hadoop.hdfs.server.federation.MiniRouterDFSCluster.RouterContext;
+import org.apache.hadoop.hdfs.server.federation.RouterConfigBuilder;
+import org.apache.hadoop.hdfs.server.federation.metrics.NamenodeBeanMetrics;
import org.apache.hadoop.hdfs.server.federation.resolver.FileSubclusterResolver;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorageReport;
import org.apache.hadoop.io.EnumSetWritable;
@@ -83,6 +86,7 @@ import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.test.GenericTestUtils;
+import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -150,7 +154,14 @@ public class TestRouterRpc {
cluster.startCluster();
// Start routers with only an RPC service
- cluster.addRouterOverrides((new RouterConfigBuilder()).rpc().build());
+ Configuration routerConf = new RouterConfigBuilder()
+ .metrics()
+ .rpc()
+ .build();
+ // We decrease the DN cache times to make the test faster
+ routerConf.setTimeDuration(
+ NamenodeBeanMetrics.DN_REPORT_CACHE_EXPIRE, 1, TimeUnit.SECONDS);
+ cluster.addRouterOverrides(routerConf);
cluster.startRouters();
// Register and verify all NNs with all routers
@@ -1032,6 +1043,32 @@ public class TestRouterRpc {
assertEquals(statsNamenode.toString(), statsRouter.toString());
}
+ @Test
+ public void testNamenodeMetrics() throws Exception {
+ final NamenodeBeanMetrics metrics =
+ router.getRouter().getNamenodeMetrics();
+ final String jsonString0 = metrics.getLiveNodes();
+
+ // We should have 12 nodes in total
+ JSONObject jsonObject = new JSONObject(jsonString0);
+ assertEquals(12, jsonObject.names().length());
+
+ // We should be caching this information
+ String jsonString1 = metrics.getLiveNodes();
+ assertEquals(jsonString0, jsonString1);
+
+ // We wait until the cached value is updated
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return !jsonString0.equals(metrics.getLiveNodes());
+ }
+ }, 500, 5 * 1000);
+
+ // The cache should be updated now
+ assertNotEquals(jsonString0, metrics.getLiveNodes());
+ }
+
/**
* Check the erasure coding policies in the Router and the Namenode.
* @return The erasure coding policies.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HDFS-13357. Improve AclException
message "Invalid ACL: only directories may have a default ACL.". Contributed
by Gabor Bota.
Posted by na...@apache.org.
HDFS-13357. Improve AclException message "Invalid ACL: only directories may have a default ACL.". Contributed by Gabor Bota.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad10cbd9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad10cbd9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad10cbd9
Branch: refs/heads/YARN-3409
Commit: ad10cbd91bcf80e63d5f98868f9258c0119db2a9
Parents: 431076f
Author: Wei-Chiu Chuang <we...@apache.org>
Authored: Thu Mar 29 07:08:22 2018 -0700
Committer: Wei-Chiu Chuang <we...@apache.org>
Committed: Thu Mar 29 07:08:22 2018 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad10cbd9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index 58dfe12..10ecc23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -281,7 +281,8 @@ public final class AclStorage {
// Only directories may have a default ACL.
if (!defaultEntries.isEmpty() && !inode.isDirectory()) {
throw new AclException(
- "Invalid ACL: only directories may have a default ACL.");
+ "Invalid ACL: only directories may have a default ACL. "
+ + "Path: " + inode.getFullPathName());
}
// Attach entries to the feature.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-7757. Refactor
NodeLabelsProvider to be more generic and reusable for node attributes
providers. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-7757. Refactor NodeLabelsProvider to be more generic and reusable for node attributes providers. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/980cc55c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/980cc55c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/980cc55c
Branch: refs/heads/YARN-3409
Commit: 980cc55c055e93f9cb471cc4e329ce9c00766f9a
Parents: d30e12d
Author: Naganarasimha <na...@apache.org>
Authored: Mon Feb 5 05:47:02 2018 +0800
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 08:17:15 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 33 +++
.../src/main/resources/yarn-default.xml | 38 ++++
.../nodemanager/NodeStatusUpdaterImpl.java | 4 +-
.../AbstractNodeDescriptorsProvider.java | 197 ++++++++++++++++
.../nodelabels/AbstractNodeLabelsProvider.java | 149 -------------
.../ConfigurationNodeLabelsProvider.java | 13 +-
.../nodelabels/NodeAttributesProvider.java | 34 +++
.../nodelabels/NodeDescriptorsProvider.java | 45 ++++
.../nodelabels/NodeDescriptorsScriptRunner.java | 84 +++++++
.../nodelabels/NodeLabelsProvider.java | 22 +-
.../ScriptBasedNodeAttributesProvider.java | 129 +++++++++++
.../ScriptBasedNodeLabelsProvider.java | 126 +++--------
.../nodemanager/nodelabels/package-info.java | 28 +++
.../TestNodeStatusUpdaterForLabels.java | 39 ++--
.../TestConfigurationNodeLabelsProvider.java | 26 ++-
.../TestScriptBasedNodeAttributesProvider.java | 223 +++++++++++++++++++
.../TestScriptBasedNodeLabelsProvider.java | 18 +-
17 files changed, 910 insertions(+), 298 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 42f2cae..730fb25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3442,6 +3442,9 @@ public class YarnConfiguration extends Configuration {
private static final String NM_NODE_LABELS_PREFIX = NM_PREFIX
+ "node-labels.";
+ private static final String NM_NODE_ATTRIBUTES_PREFIX = NM_PREFIX
+ + "node-attributes.";
+
public static final String NM_NODE_LABELS_PROVIDER_CONFIG =
NM_NODE_LABELS_PREFIX + "provider";
@@ -3452,6 +3455,9 @@ public class YarnConfiguration extends Configuration {
private static final String NM_NODE_LABELS_PROVIDER_PREFIX =
NM_NODE_LABELS_PREFIX + "provider.";
+ private static final String NM_NODE_ATTRIBUTES_PROVIDER_PREFIX =
+ NM_NODE_ATTRIBUTES_PREFIX + "provider.";
+
public static final String NM_NODE_LABELS_RESYNC_INTERVAL =
NM_NODE_LABELS_PREFIX + "resync-interval-ms";
@@ -3522,6 +3528,33 @@ public class YarnConfiguration extends Configuration {
public static final String NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS =
NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PREFIX + "opts";
+ /**
+ * Node attribute provider fetch attributes interval and timeout.
+ */
+ public static final String NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS =
+ NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "fetch-interval-ms";
+
+ public static final long
+ DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS = 10 * 60 * 1000;
+
+ public static final String NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS =
+ NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "fetch-timeout-ms";
+
+ public static final long DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS
+ = DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS * 2;
+
+ /**
+ * Script to collect node attributes.
+ */
+ private static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX =
+ NM_NODE_ATTRIBUTES_PROVIDER_PREFIX + "script.";
+
+ public static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH =
+ NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX + "path";
+
+ public static final String NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS =
+ NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PREFIX + "opts";
+
/*
* Support to view apps for given user in secure cluster.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 81b6658..2931982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2854,6 +2854,44 @@
<value>1800000</value>
</property>
+ <!-- Distributed Node Attributes Configuration -->
+ <property>
+ <description>
+ The node attribute script NM runs to collect node attributes.
+ Script output Line starting with "NODE_ATTRIBUTE:" will be
+ considered as a record of node attribute, attribute name, type
+ and value should be delimited by comma. Each of such lines
+ will be parsed to a node attribute.
+ </description>
+ <name>yarn.nodemanager.node-attributes.provider.script.path</name>
+ </property>
+
+ <property>
+ <description>
+ Command arguments passed to the node attribute script.
+ </description>
+ <name>yarn.nodemanager.node-attributes.provider.script.opts</name>
+ </property>
+
+ <property>
+ <description>
+ Time interval that determines how long NM fetches node attributes
+ from a given provider. If -1 is configured then node labels are
+ retrieved from provider only during initialization. Defaults to 10 mins.
+ </description>
+ <name>yarn.nodemanager.node-attributes.provider.fetch-interval-ms</name>
+ <value>600000</value>
+ </property>
+
+ <property>
+ <description>
+ Timeout period after which NM will interrupt the node attribute
+ provider script which queries node attributes. Defaults to 20 mins.
+ </description>
+ <name>yarn.nodemanager.node-attributes.provider.fetch-timeout-ms</name>
+ <value>1200000</value>
+ </property>
+
<property>
<description>
Timeout in seconds for YARN node graceful decommission.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 8154723..8421eea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -932,7 +932,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
@Override
public Set<NodeLabel> getNodeLabelsForRegistration() {
- Set<NodeLabel> nodeLabels = nodeLabelsProvider.getNodeLabels();
+ Set<NodeLabel> nodeLabels = nodeLabelsProvider.getDescriptors();
nodeLabels = (null == nodeLabels)
? CommonNodeLabelsManager.EMPTY_NODELABEL_SET : nodeLabels;
previousNodeLabels = nodeLabels;
@@ -967,7 +967,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
@Override
public Set<NodeLabel> getNodeLabelsForHeartbeat() {
Set<NodeLabel> nodeLabelsForHeartbeat =
- nodeLabelsProvider.getNodeLabels();
+ nodeLabelsProvider.getDescriptors();
// if the provider returns null then consider empty labels are set
nodeLabelsForHeartbeat = (nodeLabelsForHeartbeat == null)
? CommonNodeLabelsManager.EMPTY_NODELABEL_SET
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeDescriptorsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeDescriptorsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeDescriptorsProvider.java
new file mode 100644
index 0000000..088c9cb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeDescriptorsProvider.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.Timer;
+import java.util.TimerTask;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.Collections;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+
+/**
+ * Provides base implementation of NodeDescriptorsProvider with Timer and
+ * expects subclass to provide TimerTask which can fetch node descriptors.
+ */
+public abstract class AbstractNodeDescriptorsProvider<T>
+ extends AbstractService implements NodeDescriptorsProvider<T> {
+ public static final long DISABLE_NODE_DESCRIPTORS_PROVIDER_FETCH_TIMER = -1;
+
+ // Delay after which timer task are triggered to fetch node descriptors.
+ // Default interval is -1 means it is an one time task, each implementation
+ // will override this value from configuration.
+ private long intervalTime = -1;
+
+ // Timer used to schedule node descriptors fetching
+ private Timer scheduler;
+
+ protected Lock readLock = null;
+ protected Lock writeLock = null;
+
+ protected TimerTask timerTask;
+
+ private Set<T> nodeDescriptors = Collections
+ .unmodifiableSet(new HashSet<>(0));
+
+ public AbstractNodeDescriptorsProvider(String name) {
+ super(name);
+ }
+
+ public long getIntervalTime() {
+ return intervalTime;
+ }
+
+ public void setIntervalTime(long intervalMS) {
+ this.intervalTime = intervalMS;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ readLock = readWriteLock.readLock();
+ writeLock = readWriteLock.writeLock();
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ timerTask = createTimerTask();
+ timerTask.run();
+ long taskInterval = getIntervalTime();
+ if (taskInterval != DISABLE_NODE_DESCRIPTORS_PROVIDER_FETCH_TIMER) {
+ scheduler =
+ new Timer("DistributedNodeDescriptorsRunner-Timer", true);
+ // Start the timer task and then periodically at the configured interval
+ // time. Illegal values for intervalTime is handled by timer api
+ scheduler.schedule(timerTask, taskInterval, taskInterval);
+ }
+ super.serviceStart();
+ }
+
+ /**
+ * terminate the timer
+ * @throws Exception
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (scheduler != null) {
+ scheduler.cancel();
+ }
+ cleanUp();
+ super.serviceStop();
+ }
+
+ /**
+ * method for subclasses to cleanup.
+ */
+ protected abstract void cleanUp() throws Exception ;
+
+ /**
+ * @return Returns output from provider.
+ */
+ @Override
+ public Set<T> getDescriptors() {
+ readLock.lock();
+ try {
+ return this.nodeDescriptors;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public void setDescriptors(Set<T> descriptorsSet) {
+ writeLock.lock();
+ try {
+ this.nodeDescriptors = descriptorsSet;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /**
+ * Method used to determine if or not node descriptors fetching script is
+ * configured and whether it is fit to run. Returns true if following
+ * conditions are met:
+ *
+ * <ol>
+ * <li>Path to the script is not empty</li>
+ * <li>The script file exists</li>
+ * </ol>
+ *
+ * @throws IOException
+ */
+ protected void verifyConfiguredScript(String scriptPath)
+ throws IOException {
+ boolean invalidConfiguration;
+ if (scriptPath == null
+ || scriptPath.trim().isEmpty()) {
+ invalidConfiguration = true;
+ } else {
+ File f = new File(scriptPath);
+ invalidConfiguration = !f.exists() || !FileUtil.canExecute(f);
+ }
+ if (invalidConfiguration) {
+ throw new IOException(
+ "Node descriptors provider script \"" + scriptPath
+ + "\" is not configured properly. Please check whether"
+ + " the script path exists, owner and the access rights"
+ + " are suitable for NM process to execute it");
+ }
+ }
+
+ static Set<NodeLabel> convertToNodeLabelSet(String partitionNodeLabel) {
+ if (null == partitionNodeLabel) {
+ return null;
+ }
+ Set<NodeLabel> labels = new HashSet<NodeLabel>();
+ labels.add(NodeLabel.newInstance(partitionNodeLabel));
+ return labels;
+ }
+
+ /**
+ * Used only by tests to access the timer task directly
+ *
+ * @return the timer task
+ */
+ TimerTask getTimerTask() {
+ return timerTask;
+ }
+
+ @VisibleForTesting
+ public Timer getScheduler() {
+ return this.scheduler;
+ }
+
+ /**
+ * Creates a timer task which be scheduled periodically by the provider,
+ * and the task is responsible to update node descriptors to the provider.
+ * @return a timer task.
+ */
+ public abstract TimerTask createTimerTask();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeLabelsProvider.java
deleted file mode 100644
index c810654..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/AbstractNodeLabelsProvider.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
-
-import java.util.HashSet;
-import java.util.Set;
-import java.util.Timer;
-import java.util.TimerTask;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.NodeLabel;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-
-/**
- * Provides base implementation of NodeLabelsProvider with Timer and expects
- * subclass to provide TimerTask which can fetch NodeLabels
- */
-public abstract class AbstractNodeLabelsProvider extends AbstractService
- implements NodeLabelsProvider {
- public static final long DISABLE_NODE_LABELS_PROVIDER_FETCH_TIMER = -1;
-
- // Delay after which timer task are triggered to fetch NodeLabels
- protected long intervalTime;
-
- // Timer used to schedule node labels fetching
- protected Timer nodeLabelsScheduler;
-
- public static final String NODE_LABELS_SEPRATOR = ",";
-
- protected Lock readLock = null;
- protected Lock writeLock = null;
-
- protected TimerTask timerTask;
-
- protected Set<NodeLabel> nodeLabels =
- CommonNodeLabelsManager.EMPTY_NODELABEL_SET;
-
-
- public AbstractNodeLabelsProvider(String name) {
- super(name);
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- this.intervalTime =
- conf.getLong(YarnConfiguration.NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
- YarnConfiguration.DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS);
-
- ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
- readLock = readWriteLock.readLock();
- writeLock = readWriteLock.writeLock();
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- timerTask = createTimerTask();
- timerTask.run();
- if (intervalTime != DISABLE_NODE_LABELS_PROVIDER_FETCH_TIMER) {
- nodeLabelsScheduler =
- new Timer("DistributedNodeLabelsRunner-Timer", true);
- // Start the timer task and then periodically at the configured interval
- // time. Illegal values for intervalTime is handled by timer api
- nodeLabelsScheduler.scheduleAtFixedRate(timerTask, intervalTime,
- intervalTime);
- }
- super.serviceStart();
- }
-
- /**
- * terminate the timer
- * @throws Exception
- */
- @Override
- protected void serviceStop() throws Exception {
- if (nodeLabelsScheduler != null) {
- nodeLabelsScheduler.cancel();
- }
- cleanUp();
- super.serviceStop();
- }
-
- /**
- * method for subclasses to cleanup.
- */
- protected abstract void cleanUp() throws Exception ;
-
- /**
- * @return Returns output from provider.
- */
- @Override
- public Set<NodeLabel> getNodeLabels() {
- readLock.lock();
- try {
- return nodeLabels;
- } finally {
- readLock.unlock();
- }
- }
-
- protected void setNodeLabels(Set<NodeLabel> nodeLabelsSet) {
- writeLock.lock();
- try {
- nodeLabels = nodeLabelsSet;
- } finally {
- writeLock.unlock();
- }
- }
-
- static Set<NodeLabel> convertToNodeLabelSet(String partitionNodeLabel) {
- if (null == partitionNodeLabel) {
- return null;
- }
- Set<NodeLabel> labels = new HashSet<NodeLabel>();
- labels.add(NodeLabel.newInstance(partitionNodeLabel));
- return labels;
- }
-
- /**
- * Used only by tests to access the timer task directly
- *
- * @return the timer task
- */
- TimerTask getTimerTask() {
- return timerTask;
- }
-
- public abstract TimerTask createTimerTask();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
index 7490cc2..1c6af8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* Provides Node's Labels by constantly monitoring the configuration.
*/
-public class ConfigurationNodeLabelsProvider extends AbstractNodeLabelsProvider {
+public class ConfigurationNodeLabelsProvider extends NodeLabelsProvider {
private static final Logger LOG =
LoggerFactory.getLogger(ConfigurationNodeLabelsProvider.class);
@@ -38,11 +38,20 @@ public class ConfigurationNodeLabelsProvider extends AbstractNodeLabelsProvider
super("Configuration Based NodeLabels Provider");
}
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ long taskInterval = conf.getLong(
+ YarnConfiguration.NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS);
+ this.setIntervalTime(taskInterval);
+ super.serviceInit(conf);
+ }
+
private void updateNodeLabelsFromConfig(Configuration conf)
throws IOException {
String configuredNodePartition =
conf.get(YarnConfiguration.NM_PROVIDER_CONFIGURED_NODE_PARTITION, null);
- setNodeLabels(convertToNodeLabelSet(configuredNodePartition));
+ setDescriptors(convertToNodeLabelSet(configuredNodePartition));
}
private class ConfigurationMonitorTimerTask extends TimerTask {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java
new file mode 100644
index 0000000..8240024
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeAttributesProvider.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
+import java.util.Set;
+
+/**
+ * Abstract class which will be responsible for fetching the node attributes.
+ *
+ */
+public abstract class NodeAttributesProvider
+ extends AbstractNodeDescriptorsProvider<NodeAttribute> {
+
+ public NodeAttributesProvider(String name) {
+ super(name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsProvider.java
new file mode 100644
index 0000000..51608b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsProvider.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import java.util.Set;
+
+/**
+ * Interface which will be responsible for fetching node descriptors,
+ * a node descriptor could be a
+ * {@link org.apache.hadoop.yarn.api.records.NodeLabel} or a
+ * {@link org.apache.hadoop.yarn.api.records.NodeAttribute}.
+ */
+public interface NodeDescriptorsProvider<T> {
+
+ /**
+ * Provides the descriptors. The provider is expected to give same
+ * descriptors continuously until there is a change.
+ * If null is returned then an empty set is assumed by the caller.
+ *
+ * @return Set of node descriptors applicable for a node
+ */
+ Set<T> getDescriptors();
+
+ /**
+ * Sets a set of descriptors to the provider.
+ * @param descriptors node descriptors.
+ */
+ void setDescriptors(Set<T> descriptors);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsScriptRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsScriptRunner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsScriptRunner.java
new file mode 100644
index 0000000..6365f3b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeDescriptorsScriptRunner.java
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import org.apache.hadoop.util.Shell;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Set;
+import java.util.TimerTask;
+
+/**
+ * A node descriptors script runner periodically runs a script,
+ * parses the output to collect desired descriptors, and then
+ * post these descriptors to the given {@link NodeDescriptorsProvider}.
+ * @param <T> a certain type of descriptor.
+ */
+public abstract class NodeDescriptorsScriptRunner<T> extends TimerTask {
+
+ private final static Logger LOG = LoggerFactory
+ .getLogger(NodeDescriptorsScriptRunner.class);
+
+ private final Shell.ShellCommandExecutor exec;
+ private final NodeDescriptorsProvider provider;
+
+ public NodeDescriptorsScriptRunner(String scriptPath,
+ String[] scriptArgs, long scriptTimeout,
+ NodeDescriptorsProvider ndProvider) {
+ ArrayList<String> execScript = new ArrayList<>();
+ execScript.add(scriptPath);
+ if (scriptArgs != null) {
+ execScript.addAll(Arrays.asList(scriptArgs));
+ }
+ this.provider = ndProvider;
+ this.exec = new Shell.ShellCommandExecutor(
+ execScript.toArray(new String[execScript.size()]), null, null,
+ scriptTimeout);
+ }
+
+ @Override
+ public void run() {
+ try {
+ exec.execute();
+ provider.setDescriptors(parseOutput(exec.getOutput()));
+ } catch (Exception e) {
+ if (exec.isTimedOut()) {
+ LOG.warn("Node Labels script timed out, Caught exception : "
+ + e.getMessage(), e);
+ } else {
+ LOG.warn("Execution of Node Labels script failed, Caught exception : "
+ + e.getMessage(), e);
+ }
+ }
+ }
+
+ public void cleanUp() {
+ if (exec != null) {
+ Process p = exec.getProcess();
+ if (p != null) {
+ p.destroy();
+ }
+ }
+ }
+
+ abstract Set<T> parseOutput(String scriptOutput) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
index 0c076ef..9610568 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/NodeLabelsProvider.java
@@ -18,22 +18,16 @@
package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
-import java.util.Set;
-
import org.apache.hadoop.yarn.api.records.NodeLabel;
/**
- * Interface which will be responsible for fetching the labels
- *
+ * Abstract class which will be responsible for fetching the node labels.
+ *
*/
-public interface NodeLabelsProvider {
+public abstract class NodeLabelsProvider
+ extends AbstractNodeDescriptorsProvider<NodeLabel>{
- /**
- * Provides the labels. LabelProvider is expected to give same Labels
- * continuously until there is a change in labels.
- * If null is returned then Empty label set is assumed by the caller.
- *
- * @return Set of node label strings applicable for a node
- */
- public abstract Set<NodeLabel> getNodeLabels();
-}
\ No newline at end of file
+ public NodeLabelsProvider(String name) {
+ super(name);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
new file mode 100644
index 0000000..06771ba
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.TimerTask;
+
+import static org.apache.hadoop.yarn.conf.YarnConfiguration
+ .NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.
+ NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.
+ NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.
+ NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.
+ DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS;
+import static org.apache.hadoop.yarn.conf.YarnConfiguration.
+ DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS;
+
+/**
+ * Node attribute provider that periodically runs a script to collect
+ * node attributes.
+ */
+public class ScriptBasedNodeAttributesProvider extends NodeAttributesProvider{
+
+ private static final String NODE_ATTRIBUTE_PATTERN = "NODE_ATTRIBUTE:";
+ private static final String NODE_ATTRIBUTE_DELIMITER = ",";
+
+ private NodeAttributeScriptRunner runner;
+
+ public ScriptBasedNodeAttributesProvider() {
+ super(ScriptBasedNodeAttributesProvider.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ String nodeAttributeProviderScript = conf.get(
+ NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH);
+ long scriptTimeout = conf.getLong(
+ NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS,
+ DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS);
+ String[] scriptArgs = conf.getStrings(
+ NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_OPTS,
+ new String[] {});
+ verifyConfiguredScript(nodeAttributeProviderScript);
+
+ long intervalTime = conf.getLong(
+ NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS,
+ DEFAULT_NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS);
+ this.setIntervalTime(intervalTime);
+
+ this.runner = new NodeAttributeScriptRunner(nodeAttributeProviderScript,
+ scriptArgs, scriptTimeout, this);
+ }
+
+ @Override
+ protected void cleanUp() throws Exception {
+ runner.cleanUp();
+ }
+
+ @Override
+ public TimerTask createTimerTask() {
+ return runner;
+ }
+
+ private static class NodeAttributeScriptRunner extends
+ NodeDescriptorsScriptRunner<NodeAttribute> {
+
+ NodeAttributeScriptRunner(String scriptPath, String[] scriptArgs,
+ long scriptTimeout, ScriptBasedNodeAttributesProvider provider) {
+ super(scriptPath, scriptArgs, scriptTimeout, provider);
+ }
+
+ @Override
+ Set<NodeAttribute> parseOutput(String scriptOutput) throws IOException {
+ Set<NodeAttribute> attributeSet = new HashSet<>();
+ // TODO finalize format
+
+ // each line is a record of ndoe attribute like following:
+ // NODE_ATTRIBUTE:ATTRIBUTE_NAME,ATTRIBUTE_TYPE,ATTRIBUTE_VALUE
+ String[] splits = scriptOutput.split("\n");
+ for (String line : splits) {
+ String trimmedLine = line.trim();
+ if (trimmedLine.startsWith(NODE_ATTRIBUTE_PATTERN)) {
+ String nodeAttribute = trimmedLine
+ .substring(NODE_ATTRIBUTE_PATTERN.length());
+ String[] attributeStrs = nodeAttribute
+ .split(NODE_ATTRIBUTE_DELIMITER);
+ if (attributeStrs.length != 3) {
+ throw new IOException("Malformed output, expecting format "
+ + NODE_ATTRIBUTE_PATTERN + ":" + "ATTRIBUTE_NAME"
+ + NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_TYPE"
+ + NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_VALUE; but get "
+ + nodeAttribute);
+ }
+ NodeAttribute na = NodeAttribute
+ .newInstance(attributeStrs[0],
+ NodeAttributeType.valueOf(attributeStrs[1]),
+ attributeStrs[2]);
+ attributeSet.add(na);
+ }
+ }
+ return attributeSet;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
index 32f180a..c867de9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
@@ -18,19 +18,11 @@
package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
-import java.io.File;
import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Set;
-import java.util.Timer;
import java.util.TimerTask;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileUtil;
-import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -40,20 +32,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
* pattern which will be used to search node label partition from the out put of
* the NodeLabels provider script
*/
-public class ScriptBasedNodeLabelsProvider extends AbstractNodeLabelsProvider {
- /** Absolute path to the node labels script. */
- private String nodeLabelsScriptPath;
-
- /** Time after which the script should be timed out */
- private long scriptTimeout;
-
- /** ShellCommandExecutor used to execute monitoring script */
- ShellCommandExecutor shexec = null;
+public class ScriptBasedNodeLabelsProvider extends NodeLabelsProvider {
/** Pattern used for searching in the output of the node labels script */
public static final String NODE_LABEL_PARTITION_PATTERN = "NODE_PARTITION:";
- private String[] scriptArgs;
+ private NodeDescriptorsScriptRunner runner;
public ScriptBasedNodeLabelsProvider() {
super(ScriptBasedNodeLabelsProvider.class.getName());
@@ -64,48 +48,24 @@ public class ScriptBasedNodeLabelsProvider extends AbstractNodeLabelsProvider {
*/
@Override
protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- this.nodeLabelsScriptPath =
+ String nodeLabelsScriptPath =
conf.get(YarnConfiguration.NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_PATH);
- this.scriptTimeout =
+ long scriptTimeout =
conf.getLong(YarnConfiguration.NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS,
YarnConfiguration.DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_TIMEOUT_MS);
- scriptArgs = conf.getStrings(
+ String[] scriptArgs = conf.getStrings(
YarnConfiguration.NM_SCRIPT_BASED_NODE_LABELS_PROVIDER_SCRIPT_OPTS,
new String[] {});
+ verifyConfiguredScript(nodeLabelsScriptPath);
- verifyConfiguredScript();
- }
+ long taskInterval = conf.getLong(
+ YarnConfiguration.NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS);
+ this.setIntervalTime(taskInterval);
+ this.runner = new NodeLabelScriptRunner(nodeLabelsScriptPath, scriptArgs,
+ scriptTimeout, this);
- /**
- * Method used to determine if or not node labels fetching script is
- * configured and whether it is fit to run. Returns true if following
- * conditions are met:
- *
- * <ol>
- * <li>Path to Node Labels fetch script is not empty</li>
- * <li>Node Labels fetch script file exists</li>
- * </ol>
- *
- * @throws IOException
- */
- private void verifyConfiguredScript()
- throws IOException {
- boolean invalidConfiguration = false;
- if (nodeLabelsScriptPath == null
- || nodeLabelsScriptPath.trim().isEmpty()) {
- invalidConfiguration = true;
- } else {
- File f = new File(nodeLabelsScriptPath);
- invalidConfiguration = !f.exists() || !FileUtil.canExecute(f);
- }
- if (invalidConfiguration) {
- throw new IOException(
- "Distributed Node labels provider script \"" + nodeLabelsScriptPath
- + "\" is not configured properly. Please check whether the script "
- + "path exists, owner and the access rights are suitable for NM "
- + "process to execute it");
- }
+ super.serviceInit(conf);
}
/**
@@ -113,53 +73,19 @@ public class ScriptBasedNodeLabelsProvider extends AbstractNodeLabelsProvider {
*/
@Override
public void cleanUp() {
- if (shexec != null) {
- Process p = shexec.getProcess();
- if (p != null) {
- p.destroy();
- }
+ if (runner != null) {
+ runner.cleanUp();
}
}
- @Override
- public TimerTask createTimerTask() {
- return new NodeLabelsScriptRunner();
- }
-
- /**
- * Class which is used by the {@link Timer} class to periodically execute the
- * node labels script.
- */
- private class NodeLabelsScriptRunner extends TimerTask {
+ // A script runner periodically runs a script to get node labels,
+ // and sets these labels to the given provider.
+ private static class NodeLabelScriptRunner extends
+ NodeDescriptorsScriptRunner<NodeLabel> {
- private final Logger LOG =
- LoggerFactory.getLogger(NodeLabelsScriptRunner.class);
-
- public NodeLabelsScriptRunner() {
- ArrayList<String> execScript = new ArrayList<String>();
- execScript.add(nodeLabelsScriptPath);
- if (scriptArgs != null) {
- execScript.addAll(Arrays.asList(scriptArgs));
- }
- shexec = new ShellCommandExecutor(
- execScript.toArray(new String[execScript.size()]), null, null,
- scriptTimeout);
- }
-
- @Override
- public void run() {
- try {
- shexec.execute();
- setNodeLabels(fetchLabelsFromScriptOutput(shexec.getOutput()));
- } catch (Exception e) {
- if (shexec.isTimedOut()) {
- LOG.warn("Node Labels script timed out, Caught exception : "
- + e.getMessage(), e);
- } else {
- LOG.warn("Execution of Node Labels script failed, Caught exception : "
- + e.getMessage(), e);
- }
- }
+ NodeLabelScriptRunner(String scriptPath, String[] scriptArgs,
+ long scriptTimeout, ScriptBasedNodeLabelsProvider provider) {
+ super(scriptPath, scriptArgs, scriptTimeout, provider);
}
/**
@@ -170,7 +96,8 @@ public class ScriptBasedNodeLabelsProvider extends AbstractNodeLabelsProvider {
* @return true if output string has error pattern in it.
* @throws IOException
*/
- private Set<NodeLabel> fetchLabelsFromScriptOutput(String scriptOutput)
+ @Override
+ Set<NodeLabel> parseOutput(String scriptOutput)
throws IOException {
String nodePartitionLabel = null;
String[] splits = scriptOutput.split("\n");
@@ -184,4 +111,9 @@ public class ScriptBasedNodeLabelsProvider extends AbstractNodeLabelsProvider {
return convertToNodeLabelSet(nodePartitionLabel);
}
}
+
+ @Override
+ public TimerTask createTimerTask() {
+ return runner;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/package-info.java
new file mode 100644
index 0000000..190022f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package to encapsulate classes used to handle node labels and node
+ * attributes in NM.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
index 257e18c..7ef23cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdaterForLabels.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.lang.Thread.State;
import java.nio.ByteBuffer;
import java.util.Set;
+import java.util.TimerTask;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.ServerSocketUtil;
@@ -179,17 +180,27 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
}
}
- public static class DummyNodeLabelsProvider implements NodeLabelsProvider {
+ public static class DummyNodeLabelsProvider extends NodeLabelsProvider {
- private Set<NodeLabel> nodeLabels = CommonNodeLabelsManager.EMPTY_NODELABEL_SET;
+ public DummyNodeLabelsProvider() {
+ super("DummyNodeLabelsProvider");
+ // disable the fetch timer.
+ setIntervalTime(-1);
+ }
@Override
- public synchronized Set<NodeLabel> getNodeLabels() {
- return nodeLabels;
+ protected void cleanUp() throws Exception {
+ // fake implementation, nothing to cleanup
}
- synchronized void setNodeLabels(Set<NodeLabel> nodeLabels) {
- this.nodeLabels = nodeLabels;
+ @Override
+ public TimerTask createTimerTask() {
+ return new TimerTask() {
+ @Override
+ public void run() {
+ setDescriptors(CommonNodeLabelsManager.EMPTY_NODELABEL_SET);
+ }
+ };
}
}
@@ -241,18 +252,18 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
resourceTracker.resetNMHeartbeatReceiveFlag();
nm.start();
resourceTracker.waitTillRegister();
- assertNLCollectionEquals(dummyLabelsProviderRef.getNodeLabels(),
+ assertNLCollectionEquals(dummyLabelsProviderRef.getDescriptors(),
resourceTracker.labels);
resourceTracker.waitTillHeartbeat();// wait till the first heartbeat
resourceTracker.resetNMHeartbeatReceiveFlag();
// heartbeat with updated labels
- dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P"));
+ dummyLabelsProviderRef.setDescriptors(toNodeLabelSet("P"));
sendOutofBandHeartBeat();
resourceTracker.waitTillHeartbeat();
- assertNLCollectionEquals(dummyLabelsProviderRef.getNodeLabels(),
+ assertNLCollectionEquals(dummyLabelsProviderRef.getDescriptors(),
resourceTracker.labels);
resourceTracker.resetNMHeartbeatReceiveFlag();
@@ -265,7 +276,7 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
resourceTracker.labels);
// provider return with null labels
- dummyLabelsProviderRef.setNodeLabels(null);
+ dummyLabelsProviderRef.setDescriptors(null);
sendOutofBandHeartBeat();
resourceTracker.waitTillHeartbeat();
assertNotNull(
@@ -279,7 +290,7 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
// so that every sec 1 heartbeat is send.
int nullLabels = 0;
int nonNullLabels = 0;
- dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P1"));
+ dummyLabelsProviderRef.setDescriptors(toNodeLabelSet("P1"));
for (int i = 0; i < 5; i++) {
sendOutofBandHeartBeat();
resourceTracker.waitTillHeartbeat();
@@ -331,19 +342,19 @@ public class TestNodeStatusUpdaterForLabels extends NodeLabelTestBase {
};
}
};
- dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("P"));
+
YarnConfiguration conf = createNMConfigForDistributeNodeLabels();
conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:"
+ ServerSocketUtil.getPort(8040, 10));
-
nm.init(conf);
resourceTracker.resetNMHeartbeatReceiveFlag();
nm.start();
+ dummyLabelsProviderRef.setDescriptors(toNodeLabelSet("P"));
resourceTracker.waitTillHeartbeat();// wait till the first heartbeat
resourceTracker.resetNMHeartbeatReceiveFlag();
// heartbeat with invalid labels
- dummyLabelsProviderRef.setNodeLabels(toNodeLabelSet("_.P"));
+ dummyLabelsProviderRef.setDescriptors(toNodeLabelSet("_.P"));
sendOutofBandHeartBeat();
resourceTracker.waitTillHeartbeat();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeLabelsProvider.java
index 6b4d883..2acd9b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeLabelsProvider.java
@@ -98,32 +98,34 @@ public class TestConfigurationNodeLabelsProvider extends NodeLabelTestBase {
// test for ensuring labels are set during initialization of the class
nodeLabelsProvider.start();
assertNLCollectionEquals(toNodeLabelSet("A"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
// test for valid Modification
TimerTask timerTask = nodeLabelsProvider.getTimerTask();
modifyConf("X");
timerTask.run();
assertNLCollectionEquals(toNodeLabelSet("X"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
}
@Test
public void testConfigForNoTimer() throws Exception {
Configuration conf = new Configuration();
modifyConf("A");
- conf.setLong(YarnConfiguration.NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
- AbstractNodeLabelsProvider.DISABLE_NODE_LABELS_PROVIDER_FETCH_TIMER);
+ conf.setLong(YarnConfiguration
+ .NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
+ AbstractNodeDescriptorsProvider
+ .DISABLE_NODE_DESCRIPTORS_PROVIDER_FETCH_TIMER);
nodeLabelsProvider.init(conf);
nodeLabelsProvider.start();
Assert
- .assertNull(
- "Timer is not expected to be created when interval is configured as -1",
- nodeLabelsProvider.nodeLabelsScheduler);
- // Ensure that even though timer is not run, node labels are fetched at least once so
- // that NM registers/updates Labels with RM
+ .assertNull("Timer is not expected to be"
+ + " created when interval is configured as -1",
+ nodeLabelsProvider.getScheduler());
+ // Ensure that even though timer is not run, node labels
+ // are fetched at least once so that NM registers/updates Labels with RM
assertNLCollectionEquals(toNodeLabelSet("A"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
}
@Test
@@ -138,11 +140,11 @@ public class TestConfigurationNodeLabelsProvider extends NodeLabelTestBase {
// least once so
// that NM registers/updates Labels with RM
assertNLCollectionEquals(toNodeLabelSet("A"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
modifyConf("X");
Thread.sleep(1500);
assertNLCollectionEquals(toNodeLabelSet("X"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
new file mode 100644
index 0000000..58d2d20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.Shell;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Test cases for script based node attributes provider.
+ */
+public class TestScriptBasedNodeAttributesProvider {
+
+ private static File testRootDir = new File("target",
+ TestScriptBasedNodeAttributesProvider.class.getName() + "-localDir")
+ .getAbsoluteFile();
+
+ private final File nodeAttributeScript =
+ new File(testRootDir, Shell.appendScriptExtension("attributeScript"));
+
+ private ScriptBasedNodeAttributesProvider nodeAttributesProvider;
+
+ @Before
+ public void setup() {
+ testRootDir.mkdirs();
+ nodeAttributesProvider = new ScriptBasedNodeAttributesProvider();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (testRootDir.exists()) {
+ FileContext.getLocalFSFileContext()
+ .delete(new Path(testRootDir.getAbsolutePath()), true);
+ }
+ if (nodeAttributesProvider != null) {
+ nodeAttributesProvider.stop();
+ }
+ }
+
+ private Configuration getConfForNodeAttributeScript() {
+ Configuration conf = new Configuration();
+ conf.set(YarnConfiguration.NM_SCRIPT_BASED_NODE_ATTRIBUTES_PROVIDER_PATH,
+ nodeAttributeScript.getAbsolutePath());
+ // set bigger interval so that test cases can be run
+ conf.setLong(
+ YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_FETCH_INTERVAL_MS,
+ 1000);
+ conf.setLong(
+ YarnConfiguration.NM_NODE_ATTRIBUTES_PROVIDER_FETCH_TIMEOUT_MS,
+ 1000);
+ return conf;
+ }
+
+ private void writeNodeAttributeScriptFile(String scriptStr,
+ boolean setExecutable) throws IOException {
+ PrintWriter pw = null;
+ try {
+ FileUtil.setWritable(nodeAttributeScript, true);
+ FileUtil.setReadable(nodeAttributeScript, true);
+ pw = new PrintWriter(new FileOutputStream(nodeAttributeScript));
+ pw.println(scriptStr);
+ pw.flush();
+ } catch (Exception e) {
+ e.printStackTrace();
+ Assert.fail();
+ } finally {
+ if (null != pw) {
+ pw.close();
+ }
+ }
+ FileUtil.setExecutable(nodeAttributeScript, setExecutable);
+ }
+
+ @Test
+ public void testNodeAttributeScriptProvider()
+ throws IOException, InterruptedException {
+ String simpleScript = "echo NODE_ATTRIBUTE:host,STRING,host1234\n "
+ + "echo NODE_ATTRIBUTE:os,STRING,redhat_6_3\n "
+ + "echo NODE_ATTRIBUTE:ip,STRING,10.0.0.1";
+ writeNodeAttributeScriptFile(simpleScript, true);
+
+ nodeAttributesProvider.init(getConfForNodeAttributeScript());
+ nodeAttributesProvider.start();
+
+ try {
+ GenericTestUtils.waitFor(
+ () -> nodeAttributesProvider.getDescriptors().size() == 3,
+ 500, 3000);
+ } catch (TimeoutException e) {
+ Assert.fail("Expecting node attributes size is 3, but got "
+ + nodeAttributesProvider.getDescriptors().size());
+ }
+
+ Iterator<NodeAttribute> it = nodeAttributesProvider
+ .getDescriptors().iterator();
+ while (it.hasNext()) {
+ NodeAttribute att = it.next();
+ switch (att.getAttributeName()) {
+ case "host":
+ Assert.assertEquals(NodeAttributeType.STRING, att.getAttributeType());
+ Assert.assertEquals("host1234", att.getAttributeValue());
+ break;
+ case "os":
+ Assert.assertEquals(NodeAttributeType.STRING, att.getAttributeType());
+ Assert.assertEquals("redhat_6_3", att.getAttributeValue());
+ break;
+ case "ip":
+ Assert.assertEquals(NodeAttributeType.STRING, att.getAttributeType());
+ Assert.assertEquals("10.0.0.1", att.getAttributeValue());
+ break;
+ default:
+ Assert.fail("Unexpected attribute name " + att.getAttributeName());
+ break;
+ }
+ }
+ }
+
+ @Test
+ public void testInvalidScriptOutput()
+ throws IOException, InterruptedException {
+ // Script output doesn't have correct prefix.
+ String scriptContent = "echo host,STRING,host1234";
+ writeNodeAttributeScriptFile(scriptContent, true);
+
+ nodeAttributesProvider.init(getConfForNodeAttributeScript());
+ nodeAttributesProvider.start();
+
+ try {
+ GenericTestUtils.waitFor(
+ () -> nodeAttributesProvider.getDescriptors().size() == 1,
+ 500, 3000);
+ Assert.fail("This test should timeout because the provide is unable"
+ + " to parse any attributes from the script output.");
+ } catch (TimeoutException e) {
+ Assert.assertEquals(0, nodeAttributesProvider
+ .getDescriptors().size());
+ }
+ }
+
+ @Test
+ public void testMalformedScriptOutput() throws Exception{
+ // Script output has correct prefix but each line is malformed.
+ String scriptContent =
+ "echo NODE_ATTRIBUTE:host,STRING,host1234,a_extra_column";
+ writeNodeAttributeScriptFile(scriptContent, true);
+
+ nodeAttributesProvider.init(getConfForNodeAttributeScript());
+ nodeAttributesProvider.start();
+
+ // There should be no attributes found, and we should
+ // see Malformed output warnings in the log
+ try {
+ GenericTestUtils
+ .waitFor(() -> nodeAttributesProvider
+ .getDescriptors().size() == 1,
+ 500, 3000);
+ Assert.fail("This test should timeout because the provide is unable"
+ + " to parse any attributes from the script output.");
+ } catch (TimeoutException e) {
+ Assert.assertEquals(0, nodeAttributesProvider
+ .getDescriptors().size());
+ }
+ }
+
+ @Test
+ public void testFetchInterval() throws Exception {
+ // The script returns the pid (as an attribute) each time runs this script
+ String simpleScript = "echo NODE_ATTRIBUTE:pid,STRING,$$";
+ writeNodeAttributeScriptFile(simpleScript, true);
+
+ nodeAttributesProvider.init(getConfForNodeAttributeScript());
+ nodeAttributesProvider.start();
+
+ // Wait for at most 3 seconds until we get at least 1
+ // different attribute value.
+ Set<String> resultSet = new HashSet<>();
+ GenericTestUtils.waitFor(() -> {
+ Set<NodeAttribute> attributes =
+ nodeAttributesProvider.getDescriptors();
+ if (attributes != null) {
+ Assert.assertEquals(1, attributes.size());
+ resultSet.add(attributes.iterator().next().getAttributeValue());
+ return resultSet.size() > 1;
+ } else {
+ return false;
+ }
+ }, 500, 3000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/980cc55c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeLabelsProvider.java
index 1e98547..87d100f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeLabelsProvider.java
@@ -151,19 +151,21 @@ public class TestScriptBasedNodeLabelsProvider extends NodeLabelTestBase {
@Test
public void testConfigForNoTimer() throws Exception {
Configuration conf = getConfForNodeLabelScript();
- conf.setLong(YarnConfiguration.NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
- AbstractNodeLabelsProvider.DISABLE_NODE_LABELS_PROVIDER_FETCH_TIMER);
+ conf.setLong(YarnConfiguration
+ .NM_NODE_LABELS_PROVIDER_FETCH_INTERVAL_MS,
+ AbstractNodeDescriptorsProvider
+ .DISABLE_NODE_DESCRIPTORS_PROVIDER_FETCH_TIMER);
String normalScript = "echo NODE_PARTITION:X86";
writeNodeLabelsScriptFile(normalScript, true);
nodeLabelsProvider.init(conf);
nodeLabelsProvider.start();
Assert.assertNull(
"Timer is not expected to be created when interval is configured as -1",
- nodeLabelsProvider.nodeLabelsScheduler);
+ nodeLabelsProvider.getScheduler());
// Ensure that even though timer is not run script is run at least once so
// that NM registers/updates Labels with RM
assertNLCollectionEquals(toNodeLabelSet("X86"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
}
@Test
@@ -185,25 +187,25 @@ public class TestScriptBasedNodeLabelsProvider extends NodeLabelTestBase {
Assert.assertNull(
"Node Label Script runner should return null when script doesnt "
+ "give any Labels output",
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
writeNodeLabelsScriptFile(normalScript, true);
timerTask.run();
assertNLCollectionEquals(toNodeLabelSet("Windows"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
// multiple lines with partition tag then the last line's partition info
// needs to be taken.
writeNodeLabelsScriptFile(scrptWithMultipleLinesHavingNodeLabels, true);
timerTask.run();
assertNLCollectionEquals(toNodeLabelSet("JDK1_6"),
- nodeLabelsProvider.getNodeLabels());
+ nodeLabelsProvider.getDescriptors());
// timeout script.
writeNodeLabelsScriptFile(timeOutScript, true);
timerTask.run();
Assert.assertNotEquals("Node Labels should not be set after timeout ",
- toNodeLabelSet("ALL"), nodeLabelsProvider.getNodeLabels());
+ toNodeLabelSet("ALL"), nodeLabelsProvider.getDescriptors());
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-8075. DShell does not fail
when we ask more GPUs than available even though AM throws
'InvalidResourceRequestException'. Contributed by Wangda Tan.
Posted by na...@apache.org.
YARN-8075. DShell does not fail when we ask more GPUs than available even though AM throws 'InvalidResourceRequestException'. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1e378d0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1e378d0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1e378d0
Branch: refs/heads/YARN-3409
Commit: d1e378d02bdaec2f078ccc89698e7959ffcc71b3
Parents: 2a2ef15
Author: Sunil G <su...@apache.org>
Authored: Wed Mar 28 08:03:18 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Mar 28 08:03:18 2018 +0530
----------------------------------------------------------------------
.../yarn/applications/distributedshell/ApplicationMaster.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1e378d0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 5c10775..75f4073 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -1147,7 +1147,6 @@ public class ApplicationMaster {
public void onError(Throwable e) {
LOG.error("Error in RMCallbackHandler: ", e);
done = true;
- amRMClient.stop();
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: HDFS-13314. NameNode should
optionally exit if it detects FsImage corruption. Contributed by Arpit
Agarwal.
Posted by na...@apache.org.
HDFS-13314. NameNode should optionally exit if it detects FsImage corruption. Contributed by Arpit Agarwal.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a991e899
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a991e899
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a991e899
Branch: refs/heads/YARN-3409
Commit: a991e899fb9f98d2089f37ac9ac7c485d3bbb959
Parents: 081c350
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Mar 28 11:37:34 2018 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Mar 28 12:49:17 2018 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSImage.java | 29 +++++++++--
.../server/namenode/FSImageFormatProtobuf.java | 31 ++++++++---
.../snapshot/FSImageFormatPBSnapshot.java | 55 ++++++++++++++++++--
3 files changed, 101 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a991e899/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index e758108..dd7df5a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -35,6 +35,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -68,6 +69,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.util.Canceler;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Time;
import com.google.common.base.Joiner;
@@ -86,6 +88,10 @@ public class FSImage implements Closeable {
protected FSEditLog editLog = null;
private boolean isUpgradeFinalized = false;
+ // If true, then image corruption was detected. The NameNode process will
+ // exit immediately after saving the image.
+ private AtomicBoolean exitAfterSave = new AtomicBoolean(false);
+
protected NNStorage storage;
/**
@@ -954,8 +960,14 @@ public class FSImage implements Closeable {
FSImageFormatProtobuf.Saver saver = new FSImageFormatProtobuf.Saver(context);
FSImageCompression compression = FSImageCompression.createCompression(conf);
- saver.save(newFile, compression);
-
+ long numErrors = saver.save(newFile, compression);
+ if (numErrors > 0) {
+ // The image is likely corrupted.
+ LOG.error("Detected " + numErrors + " errors while saving FsImage " +
+ dstFile);
+ exitAfterSave.set(true);
+ }
+
MD5FileUtils.saveMD5File(dstFile, saver.getSavedDigest());
storage.setMostRecentCheckpointInfo(txid, Time.now());
}
@@ -1117,6 +1129,12 @@ public class FSImage implements Closeable {
}
//Update NameDirSize Metric
getStorage().updateNameDirSize();
+
+ if (exitAfterSave.get()) {
+ LOG.fatal("NameNode process will exit now... The saved FsImage " +
+ nnf + " is potentially corrupted.");
+ ExitUtil.terminate(-1);
+ }
}
/**
@@ -1184,8 +1202,11 @@ public class FSImage implements Closeable {
// Since we now have a new checkpoint, we can clean up some
// old edit logs and checkpoints.
- purgeOldStorage(nnf);
- archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW);
+ // Do not purge anything if we just wrote a corrupted FsImage.
+ if (!exitAfterSave.get()) {
+ purgeOldStorage(nnf);
+ archivalManager.purgeCheckpoints(NameNodeFile.IMAGE_NEW);
+ }
} finally {
// Notify any threads waiting on the checkpoint to be canceled
// that it is complete.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a991e899/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
index cd5a5526..4ac20ad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatProtobuf.java
@@ -444,15 +444,22 @@ public final class FSImageFormatProtobuf {
sectionOutputStream.flush();
}
- void save(File file, FSImageCompression compression) throws IOException {
+ /**
+ * @return number of non-fatal errors detected while writing the image.
+ * @throws IOException on fatal error.
+ */
+ long save(File file, FSImageCompression compression) throws IOException {
FileOutputStream fout = new FileOutputStream(file);
fileChannel = fout.getChannel();
try {
LOG.info("Saving image file {} using {}", file, compression);
long startTime = monotonicNow();
- saveInternal(fout, compression, file.getAbsolutePath());
- LOG.info("Image file {} of size {} bytes saved in {} seconds.", file,
- file.length(), (monotonicNow() - startTime) / 1000);
+ long numErrors = saveInternal(
+ fout, compression, file.getAbsolutePath());
+ LOG.info("Image file {} of size {} bytes saved in {} seconds {}.", file,
+ file.length(), (monotonicNow() - startTime) / 1000,
+ (numErrors > 0 ? (" with" + numErrors + " errors") : ""));
+ return numErrors;
} finally {
fout.close();
}
@@ -476,7 +483,11 @@ public final class FSImageFormatProtobuf {
saver.serializeFilesUCSection(sectionOutputStream);
}
- private void saveSnapshots(FileSummary.Builder summary) throws IOException {
+ /**
+ * @return number of non-fatal errors detected while saving the image.
+ * @throws IOException on fatal error.
+ */
+ private long saveSnapshots(FileSummary.Builder summary) throws IOException {
FSImageFormatPBSnapshot.Saver snapshotSaver = new FSImageFormatPBSnapshot.Saver(
this, summary, context, context.getSourceNamesystem());
@@ -487,9 +498,14 @@ public final class FSImageFormatProtobuf {
snapshotSaver.serializeSnapshotDiffSection(sectionOutputStream);
}
snapshotSaver.serializeINodeReferenceSection(sectionOutputStream);
+ return snapshotSaver.getNumImageErrors();
}
- private void saveInternal(FileOutputStream fout,
+ /**
+ * @return number of non-fatal errors detected while writing the FsImage.
+ * @throws IOException on fatal error.
+ */
+ private long saveInternal(FileOutputStream fout,
FSImageCompression compression, String filePath) throws IOException {
StartupProgress prog = NameNode.getStartupProgress();
MessageDigest digester = MD5Hash.getDigester();
@@ -528,7 +544,7 @@ public final class FSImageFormatProtobuf {
step = new Step(StepType.INODES, filePath);
prog.beginStep(Phase.SAVING_CHECKPOINT, step);
saveInodes(b);
- saveSnapshots(b);
+ long numErrors = saveSnapshots(b);
prog.endStep(Phase.SAVING_CHECKPOINT, step);
step = new Step(StepType.DELEGATION_TOKENS, filePath);
@@ -551,6 +567,7 @@ public final class FSImageFormatProtobuf {
saveFileSummary(underlyingOutputStream, summary);
underlyingOutputStream.close();
savedDigest = new MD5Hash(digester.digest());
+ return numErrors;
}
private void saveSecretManagerSection(FileSummary.Builder summary)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a991e899/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index 4a5cead..2157554 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -1,4 +1,4 @@
-/**
+ /**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -49,6 +50,7 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.namenode.AclEntryStatusFormat;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory;
+import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.LoaderContext;
@@ -415,6 +417,7 @@ public class FSImageFormatPBSnapshot {
private final FileSummary.Builder headers;
private final FSImageFormatProtobuf.Saver parent;
private final SaveNamespaceContext context;
+ private long numImageErrors;
public Saver(FSImageFormatProtobuf.Saver parent,
FileSummary.Builder headers, SaveNamespaceContext context,
@@ -423,6 +426,7 @@ public class FSImageFormatPBSnapshot {
this.headers = headers;
this.context = context;
this.fsn = fsn;
+ this.numImageErrors = 0;
}
/**
@@ -471,15 +475,17 @@ public class FSImageFormatPBSnapshot {
throws IOException {
final List<INodeReference> refList = parent.getSaverContext()
.getRefList();
+ long i = 0;
for (INodeReference ref : refList) {
- INodeReferenceSection.INodeReference.Builder rb = buildINodeReference(ref);
+ INodeReferenceSection.INodeReference.Builder rb =
+ buildINodeReference(ref, i++);
rb.build().writeDelimitedTo(out);
}
parent.commitSection(headers, SectionName.INODE_REFERENCE);
}
private INodeReferenceSection.INodeReference.Builder buildINodeReference(
- INodeReference ref) throws IOException {
+ final INodeReference ref, final long refIndex) throws IOException {
INodeReferenceSection.INodeReference.Builder rb =
INodeReferenceSection.INodeReference.newBuilder().
setReferredId(ref.getId());
@@ -489,6 +495,16 @@ public class FSImageFormatPBSnapshot {
} else if (ref instanceof DstReference) {
rb.setDstSnapshotId(ref.getDstSnapshotId());
}
+
+ if (fsn.getFSDirectory().getInode(ref.getId()) == null) {
+ FSImage.LOG.error(
+ "FSImageFormatPBSnapshot: Missing referred INodeId " +
+ ref.getId() + " for INodeReference index " + refIndex +
+ "; path=" + ref.getFullPathName() +
+ "; parent=" + (ref.getParent() == null ? "null" :
+ ref.getParent().getFullPathName()));
+ ++numImageErrors;
+ }
return rb;
}
@@ -583,7 +599,23 @@ public class FSImageFormatPBSnapshot {
List<INode> created = diff.getChildrenDiff().getCreatedUnmodifiable();
db.setCreatedListSize(created.size());
List<INode> deleted = diff.getChildrenDiff().getDeletedUnmodifiable();
+ INode previousNode = null;
+ boolean misordered = false;
for (INode d : deleted) {
+ // getBytes() may return null below, and that is okay.
+ final int result = previousNode == null ? -1 :
+ previousNode.compareTo(d.getLocalNameBytes());
+ if (result == 0) {
+ FSImage.LOG.error(
+ "Name '" + d.getLocalName() + "' is repeated in the " +
+ "'deleted' difflist of directory " +
+ dir.getFullPathName() + ", INodeId=" + dir.getId());
+ ++numImageErrors;
+ } else if (result > 0 && !misordered) {
+ misordered = true;
+ ++numImageErrors;
+ }
+ previousNode = d;
if (d.isReference()) {
refList.add(d.asReference());
db.addDeletedINodeRef(refList.size() - 1);
@@ -591,11 +623,28 @@ public class FSImageFormatPBSnapshot {
db.addDeletedINode(d.getId());
}
}
+ if (misordered) {
+ FSImage.LOG.error(
+ "Misordered entries in the 'deleted' difflist of directory " +
+ dir.getFullPathName() + ", INodeId=" + dir.getId() +
+ ". The full list is " +
+ Arrays.toString(deleted.toArray()));
+ }
db.build().writeDelimitedTo(out);
saveCreatedList(created, out);
}
}
}
+
+
+ /**
+ * Number of non-fatal errors detected while writing the
+ * SnapshotDiff and INodeReference sections.
+ * @return the number of non-fatal errors detected.
+ */
+ public long getNumImageErrors() {
+ return numImageErrors;
+ }
}
private FSImageFormatPBSnapshot(){}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-8077. The vmemLimit parameter
in ContainersMonitorImpl#isProcessTreeOverLimit is confusing. Contributed by
Sen Zhao.
Posted by na...@apache.org.
YARN-8077. The vmemLimit parameter in ContainersMonitorImpl#isProcessTreeOverLimit is confusing. Contributed by Sen Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdee0a4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdee0a4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdee0a4f
Branch: refs/heads/YARN-3409
Commit: cdee0a4f840868d8b8acac15e62da2ab337618c7
Parents: 411993f
Author: Miklos Szegedi <sz...@apache.org>
Authored: Wed Mar 28 09:05:25 2018 -0700
Committer: Miklos Szegedi <sz...@apache.org>
Committed: Wed Mar 28 09:05:25 2018 -0700
----------------------------------------------------------------------
.../containermanager/monitor/ContainersMonitorImpl.java | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdee0a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 33986a0..35015c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -367,7 +367,7 @@ public class ContainersMonitorImpl extends AbstractService implements
* @param curMemUsageOfAgedProcesses
* Memory usage of processes older than an iteration in a container
* tree
- * @param vmemLimit
+ * @param memLimit
* The limit specified for the container
* @return true if the memory usage is more than twice the specified limit,
* or if processes in the tree, older than this thread's monitoring
@@ -376,18 +376,18 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean isProcessTreeOverLimit(String containerId,
long currentMemUsage,
long curMemUsageOfAgedProcesses,
- long vmemLimit) {
+ long memLimit) {
boolean isOverLimit = false;
- if (currentMemUsage > (2 * vmemLimit)) {
+ if (currentMemUsage > (2 * memLimit)) {
LOG.warn("Process tree for container: " + containerId
- + " running over twice " + "the configured limit. Limit=" + vmemLimit
+ + " running over twice " + "the configured limit. Limit=" + memLimit
+ ", current usage = " + currentMemUsage);
isOverLimit = true;
- } else if (curMemUsageOfAgedProcesses > vmemLimit) {
+ } else if (curMemUsageOfAgedProcesses > memLimit) {
LOG.warn("Process tree for container: " + containerId
+ " has processes older than 1 "
- + "iteration running over the configured limit. Limit=" + vmemLimit
+ + "iteration running over the configured limit. Limit=" + memLimit
+ ", current usage = " + curMemUsageOfAgedProcesses);
isOverLimit = true;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-7794. SLSRunner is not
loading timeline service jars, causing failure. (Yufei Gu via Haibo Chen)
Posted by na...@apache.org.
YARN-7794. SLSRunner is not loading timeline service jars, causing failure. (Yufei Gu via Haibo Chen)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcea5a4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcea5a4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcea5a4d
Branch: refs/heads/YARN-3409
Commit: fcea5a4d798a517443afd0b441883ccc4e9136ba
Parents: 82665a7
Author: Haibo Chen <ha...@apache.org>
Authored: Mon Mar 26 14:30:11 2018 -0700
Committer: Haibo Chen <ha...@apache.org>
Committed: Mon Mar 26 14:30:11 2018 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcea5a4d/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index e069a4c..5a9de89 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -77,6 +77,8 @@ function parse_args()
function calculate_classpath
{
hadoop_add_to_classpath_tools hadoop-sls
+
+ hadoop_add_classpath "${HADOOP_YARN_HOME}/${YARN_DIR}/timelineservice"'/*'
}
function run_simulation() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-15299. Bump Jackson 2
version to Jackson 2.9.x.
Posted by na...@apache.org.
HADOOP-15299. Bump Jackson 2 version to Jackson 2.9.x.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82665a78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82665a78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82665a78
Branch: refs/heads/YARN-3409
Commit: 82665a7887a4bbb3afbc257bec31089173f3a969
Parents: 24bc5e0
Author: Sean Mackrory <ma...@apache.org>
Authored: Thu Mar 8 11:23:23 2018 -0700
Committer: Sean Mackrory <ma...@apache.org>
Committed: Mon Mar 26 14:05:15 2018 -0600
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
.../java/org/apache/hadoop/tools/rumen/state/StatePool.java | 8 ++++----
.../java/org/apache/hadoop/tools/rumen/TestHistograms.java | 3 ++-
.../yarn/client/api/impl/FileSystemTimelineWriter.java | 4 +++-
.../java/org/apache/hadoop/yarn/server/timeline/LogInfo.java | 3 ++-
.../hadoop/yarn/server/timeline/PluginStoreTestUtils.java | 3 ++-
.../org/apache/hadoop/yarn/server/timeline/TestLogInfo.java | 6 ++++--
7 files changed, 18 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index dcbc484..c093927 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -69,7 +69,7 @@
<!-- jackson versions -->
<jackson.version>1.9.13</jackson.version>
- <jackson2.version>2.7.8</jackson2.version>
+ <jackson2.version>2.9.4</jackson2.version>
<!-- SLF4J version -->
<slf4j.version>1.7.25</slf4j.version>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
index 41eb90d..ab6f894 100644
--- a/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
+++ b/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/state/StatePool.java
@@ -18,11 +18,11 @@
package org.apache.hadoop.tools.rumen.state;
import java.io.DataInput;
-import java.io.DataInputStream;
import java.io.DataOutput;
-import java.io.DataOutputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.Calendar;
@@ -216,7 +216,7 @@ public class StatePool {
// register the module with the object-mapper
mapper.registerModule(module);
- JsonParser parser = mapper.getFactory().createParser((DataInputStream)in);
+ JsonParser parser = mapper.getFactory().createParser((InputStream)in);
StatePool statePool = mapper.readValue(parser, StatePool.class);
this.setStates(statePool.getStates());
parser.close();
@@ -285,7 +285,7 @@ public class StatePool {
JsonFactory outFactory = outMapper.getFactory();
JsonGenerator jGen =
- outFactory.createGenerator((DataOutputStream)out, JsonEncoding.UTF8);
+ outFactory.createGenerator((OutputStream)out, JsonEncoding.UTF8);
jGen.useDefaultPrettyPrinter();
jGen.writeObject(this);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
index 52caaf5..1872519 100644
--- a/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
+++ b/hadoop-tools/hadoop-rumen/src/test/java/org/apache/hadoop/tools/rumen/TestHistograms.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.tools.rumen;
import java.io.IOException;
+import java.io.OutputStream;
import java.util.List;
import com.fasterxml.jackson.core.JsonEncoding;
@@ -142,7 +143,7 @@ public class TestHistograms {
ObjectMapper mapper = new ObjectMapper();
JsonFactory factory = mapper.getFactory();
FSDataOutputStream ostream = lfs.create(goldFilePath, true);
- JsonGenerator gen = factory.createGenerator(ostream,
+ JsonGenerator gen = factory.createGenerator((OutputStream)ostream,
JsonEncoding.UTF8);
gen.useDefaultPrettyPrinter();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
index b7bb48e..c00a0b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/FileSystemTimelineWriter.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.Closeable;
import java.io.Flushable;
import java.io.IOException;
+import java.io.OutputStream;
import java.net.URI;
import java.util.ArrayList;
import java.util.HashMap;
@@ -374,7 +375,8 @@ public class FileSystemTimelineWriter extends TimelineWriter{
protected void prepareForWrite() throws IOException{
this.stream = createLogFileStream(fs, logPath);
- this.jsonGenerator = new JsonFactory().createGenerator(stream);
+ this.jsonGenerator = new JsonFactory().createGenerator(
+ (OutputStream)stream);
this.jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
this.lastModifiedTime = Time.monotonicNow();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
index 57dbdcb..fa3c853 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/main/java/org/apache/hadoop/yarn/server/timeline/LogInfo.java
@@ -40,6 +40,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.io.InputStream;
import java.util.ArrayList;
abstract class LogInfo {
@@ -140,7 +141,7 @@ abstract class LogInfo {
try {
in.seek(offset);
try {
- parser = jsonFactory.createParser(in);
+ parser = jsonFactory.createParser((InputStream)in);
parser.configure(JsonParser.Feature.AUTO_CLOSE_SOURCE, false);
} catch (IOException e) {
// if app hasn't completed then there may be errors due to the
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
index 72f5a27..4609cb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/PluginStoreTestUtils.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import java.io.IOException;
+import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.EnumSet;
@@ -232,7 +233,7 @@ public class PluginStoreTestUtils {
FileSystem fs) throws IOException {
FSDataOutputStream outStream = createLogFile(logPath, fs);
JsonGenerator jsonGenerator
- = new JsonFactory().createGenerator(outStream);
+ = new JsonFactory().createGenerator((OutputStream)outStream);
jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
ObjectMapper objMapper = createObjectMapper();
for (TimelineEntity entity : entities.getEntities()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82665a78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
index f880faf..bccf4b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timeline-pluginstorage/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLogInfo.java
@@ -39,6 +39,7 @@ import org.junit.Before;
import org.junit.Test;
import java.io.IOException;
+import java.io.OutputStream;
import java.nio.charset.Charset;
import java.util.EnumSet;
@@ -232,7 +233,8 @@ public class TestLogInfo {
throws IOException {
if (outStream == null) {
outStream = PluginStoreTestUtils.createLogFile(logPath, fs);
- jsonGenerator = new JsonFactory().createGenerator(outStream);
+ jsonGenerator = new JsonFactory().createGenerator(
+ (OutputStream)outStream);
jsonGenerator.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
}
for (TimelineEntity entity : entities.getEntities()) {
@@ -248,7 +250,7 @@ public class TestLogInfo {
}
// Write domain uses its own json generator to isolate from entity writers
JsonGenerator jsonGeneratorLocal
- = new JsonFactory().createGenerator(outStreamDomain);
+ = new JsonFactory().createGenerator((OutputStream)outStreamDomain);
jsonGeneratorLocal.setPrettyPrinter(new MinimalPrettyPrinter("\n"));
objMapper.writeValue(jsonGeneratorLocal, domain);
outStreamDomain.hflush();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: MAPREDUCE-6441. Improve temporary
directory name generation in LocalDistributedCacheManager for concurrent
processes (wattsinabox, rchiang, haibochen via rkanter)
Posted by na...@apache.org.
MAPREDUCE-6441. Improve temporary directory name generation in LocalDistributedCacheManager for concurrent processes (wattsinabox, rchiang, haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/edb202e4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/edb202e4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/edb202e4
Branch: refs/heads/YARN-3409
Commit: edb202e4934be750e43103c047752b97c5eafc94
Parents: fcea5a4
Author: Robert Kanter <rk...@apache.org>
Authored: Mon Mar 26 14:55:53 2018 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Mon Mar 26 14:55:53 2018 -0700
----------------------------------------------------------------------
.../mapred/LocalDistributedCacheManager.java | 13 ++-
.../apache/hadoop/mapred/LocalJobRunner.java | 2 +-
.../TestLocalDistributedCacheManager.java | 95 +++++++++++++++-----
3 files changed, 78 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edb202e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
index 2a14ec3..bcf73d1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
@@ -37,7 +37,7 @@ import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadFactory;
-import java.util.concurrent.atomic.AtomicLong;
+import java.util.UUID;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
@@ -82,7 +82,7 @@ class LocalDistributedCacheManager {
* @param conf
* @throws IOException
*/
- public void setup(JobConf conf) throws IOException {
+ public void setup(JobConf conf, JobID jobId) throws IOException {
File workDir = new File(System.getProperty("user.dir"));
// Generate YARN local resources objects corresponding to the distributed
@@ -91,9 +91,7 @@ class LocalDistributedCacheManager {
new LinkedHashMap<String, LocalResource>();
MRApps.setupDistributedCache(conf, localResources);
// Generating unique numbers for FSDownload.
- AtomicLong uniqueNumberGenerator =
- new AtomicLong(System.currentTimeMillis());
-
+
// Find which resources are to be put on the local classpath
Map<String, Path> classpaths = new HashMap<String, Path>();
Path[] archiveClassPaths = DistributedCache.getArchiveClassPaths(conf);
@@ -124,9 +122,10 @@ class LocalDistributedCacheManager {
Path destPath = localDirAllocator.getLocalPathForWrite(".", conf);
Map<LocalResource, Future<Path>> resourcesToPaths = Maps.newHashMap();
for (LocalResource resource : localResources.values()) {
+ Path destPathForDownload = new Path(destPath,
+ jobId.toString() + "_" + UUID.randomUUID().toString());
Callable<Path> download =
- new FSDownload(localFSFileContext, ugi, conf, new Path(destPath,
- Long.toString(uniqueNumberGenerator.incrementAndGet())),
+ new FSDownload(localFSFileContext, ugi, conf, destPathForDownload,
resource);
Future<Path> future = exec.submit(download);
resourcesToPaths.put(resource, future);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edb202e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
index 5e7a250..2ab4e76 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalJobRunner.java
@@ -169,7 +169,7 @@ public class LocalJobRunner implements ClientProtocol {
// Manage the distributed cache. If there are files to be copied,
// this will trigger localFile to be re-written again.
localDistributedCacheManager = new LocalDistributedCacheManager();
- localDistributedCacheManager.setup(conf);
+ localDistributedCacheManager.setup(conf, jobid);
// Write out configuration file. Instead of copying it from
// systemJobFile, we re-write it, since setup(), above, may have
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edb202e4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
index d2814e9..fa60e2d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
@@ -32,6 +32,13 @@ import java.io.IOException;
import java.net.URI;
import java.util.HashMap;
import java.util.Map;
+import java.util.ArrayList;
+import java.util.concurrent.Callable;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -54,37 +61,37 @@ import org.mockito.stubbing.Answer;
@SuppressWarnings("deprecation")
public class TestLocalDistributedCacheManager {
- private static FileSystem mockfs;
-
+ private static FileSystem mockfs;
+
public static class MockFileSystem extends FilterFileSystem {
public MockFileSystem() {
super(mockfs);
}
}
-
+
private File localDir;
-
+
private static void delete(File file) throws IOException {
- if (file.getAbsolutePath().length() < 5) {
+ if (file.getAbsolutePath().length() < 5) {
throw new IllegalArgumentException(
"Path [" + file + "] is too short, not deleting");
}
- if (file.exists()) {
+ if (file.exists()) {
if (file.isDirectory()) {
File[] children = file.listFiles();
if (children != null) {
for (File child : children) {
delete(child);
- }
- }
- }
+ }
+ }
+ }
if (!file.delete()) {
throw new RuntimeException(
"Could not delete path [" + file + "]");
}
}
}
-
+
@Before
public void setup() throws Exception {
mockfs = mock(FileSystem.class);
@@ -93,7 +100,7 @@ public class TestLocalDistributedCacheManager {
delete(localDir);
localDir.mkdirs();
}
-
+
@After
public void cleanup() throws Exception {
delete(localDir);
@@ -120,9 +127,10 @@ public class TestLocalDistributedCacheManager {
@Test
public void testDownload() throws Exception {
+ JobID jobId = new JobID();
JobConf conf = new JobConf();
conf.setClass("fs.mock.impl", MockFileSystem.class, FileSystem.class);
-
+
URI mockBase = new URI("mock://test-nn1/");
when(mockfs.getUri()).thenReturn(mockBase);
Path working = new Path("mock://test-nn1/user/me/");
@@ -137,14 +145,14 @@ public class TestLocalDistributedCacheManager {
final URI file = new URI("mock://test-nn1/user/me/file.txt#link");
final Path filePath = new Path(file);
File link = new File("link");
-
+
when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
@Override
public FileStatus answer(InvocationOnMock args) throws Throwable {
Path p = (Path)args.getArguments()[0];
if("file.txt".equals(p.getName())) {
- return new FileStatus(201, false, 1, 500, 101, 101,
- FsPermission.getDefault(), "me", "me", filePath);
+ return new FileStatus(201, false, 1, 500, 101, 101,
+ FsPermission.getDefault(), "me", "me", filePath);
} else {
throw new FileNotFoundException(p+" not supported by mocking");
}
@@ -176,7 +184,7 @@ public class TestLocalDistributedCacheManager {
conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath());
LocalDistributedCacheManager manager = new LocalDistributedCacheManager();
try {
- manager.setup(conf);
+ manager.setup(conf, jobId);
assertTrue(link.exists());
} finally {
manager.close();
@@ -186,9 +194,10 @@ public class TestLocalDistributedCacheManager {
@Test
public void testEmptyDownload() throws Exception {
+ JobID jobId = new JobID();
JobConf conf = new JobConf();
conf.setClass("fs.mock.impl", MockFileSystem.class, FileSystem.class);
-
+
URI mockBase = new URI("mock://test-nn1/");
when(mockfs.getUri()).thenReturn(mockBase);
Path working = new Path("mock://test-nn1/user/me/");
@@ -199,7 +208,7 @@ public class TestLocalDistributedCacheManager {
return (Path) args.getArguments()[0];
}
});
-
+
when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
@Override
public FileStatus answer(InvocationOnMock args) throws Throwable {
@@ -221,7 +230,7 @@ public class TestLocalDistributedCacheManager {
conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath());
LocalDistributedCacheManager manager = new LocalDistributedCacheManager();
try {
- manager.setup(conf);
+ manager.setup(conf, jobId);
} finally {
manager.close();
}
@@ -230,9 +239,10 @@ public class TestLocalDistributedCacheManager {
@Test
public void testDuplicateDownload() throws Exception {
+ JobID jobId = new JobID();
JobConf conf = new JobConf();
conf.setClass("fs.mock.impl", MockFileSystem.class, FileSystem.class);
-
+
URI mockBase = new URI("mock://test-nn1/");
when(mockfs.getUri()).thenReturn(mockBase);
Path working = new Path("mock://test-nn1/user/me/");
@@ -247,14 +257,14 @@ public class TestLocalDistributedCacheManager {
final URI file = new URI("mock://test-nn1/user/me/file.txt#link");
final Path filePath = new Path(file);
File link = new File("link");
-
+
when(mockfs.getFileStatus(any(Path.class))).thenAnswer(new Answer<FileStatus>() {
@Override
public FileStatus answer(InvocationOnMock args) throws Throwable {
Path p = (Path)args.getArguments()[0];
if("file.txt".equals(p.getName())) {
- return new FileStatus(201, false, 1, 500, 101, 101,
- FsPermission.getDefault(), "me", "me", filePath);
+ return new FileStatus(201, false, 1, 500, 101, 101,
+ FsPermission.getDefault(), "me", "me", filePath);
} else {
throw new FileNotFoundException(p+" not supported by mocking");
}
@@ -287,11 +297,48 @@ public class TestLocalDistributedCacheManager {
conf.set(MRConfig.LOCAL_DIR, localDir.getAbsolutePath());
LocalDistributedCacheManager manager = new LocalDistributedCacheManager();
try {
- manager.setup(conf);
+ manager.setup(conf, jobId);
assertTrue(link.exists());
} finally {
manager.close();
}
assertFalse(link.exists());
}
+
+ /**
+ * This test tries to replicate the issue with the previous version of
+ * {@ref LocalDistributedCacheManager} when the resulting timestamp is
+ * identical as that in another process. Unfortunately, it is difficult
+ * to mimic such behavior in a single process unit test. And mocking
+ * the unique id (timestamp previously, UUID otherwise) won't prove the
+ * validity of one approach over the other.
+ */
+ @Test
+ public void testMultipleCacheSetup() throws Exception {
+ JobID jobId = new JobID();
+ JobConf conf = new JobConf();
+ LocalDistributedCacheManager manager = new LocalDistributedCacheManager();
+
+ final int threadCount = 10;
+ final CyclicBarrier barrier = new CyclicBarrier(threadCount);
+
+ ArrayList<Callable<Void>> setupCallable = new ArrayList<>();
+ for (int i = 0; i < threadCount; ++i) {
+ setupCallable.add(() -> {
+ barrier.await();
+ manager.setup(conf, jobId);
+ return null;
+ });
+ }
+
+ ExecutorService ePool = Executors.newFixedThreadPool(threadCount);
+ try {
+ for (Future<Void> future : ePool.invokeAll(setupCallable)) {
+ future.get();
+ }
+ } finally {
+ ePool.shutdown();
+ manager.close();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: HADOOP-12862. LDAP Group Mapping
over SSL can not specify trust store. Contributed by Wei-Chiu Chuang and
Konstantin Shvachko.
Posted by na...@apache.org.
HADOOP-12862. LDAP Group Mapping over SSL can not specify trust store. Contributed by Wei-Chiu Chuang and Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2216bde3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2216bde3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2216bde3
Branch: refs/heads/YARN-3409
Commit: 2216bde322961c0fe33b5822510880a65d5c45fd
Parents: 2c6cfad
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Mar 29 17:13:18 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Thu Mar 29 17:16:23 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 2 +-
.../hadoop/security/LdapGroupsMapping.java | 90 +++++++++++++++++---
.../src/main/resources/core-default.xml | 21 +++++
.../src/site/markdown/GroupsMapping.md | 3 +
4 files changed, 105 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2216bde3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 25fd656a..78a2e9f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -2335,7 +2335,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
* @return password or null if not found
* @throws IOException
*/
- protected char[] getPasswordFromCredentialProviders(String name)
+ public char[] getPasswordFromCredentialProviders(String name)
throws IOException {
char[] pass = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2216bde3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
index babfa38..6beaa9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/LdapGroupsMapping.java
@@ -109,6 +109,27 @@ public class LdapGroupsMapping
public static final String LDAP_KEYSTORE_PASSWORD_FILE_KEY = LDAP_KEYSTORE_PASSWORD_KEY + ".file";
public static final String LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT = "";
+
+ /**
+ * File path to the location of the SSL truststore to use
+ */
+ public static final String LDAP_TRUSTSTORE_KEY = LDAP_CONFIG_PREFIX +
+ ".ssl.truststore";
+
+ /**
+ * The key of the credential entry containing the password for
+ * the LDAP SSL truststore
+ */
+ public static final String LDAP_TRUSTSTORE_PASSWORD_KEY =
+ LDAP_CONFIG_PREFIX +".ssl.truststore.password";
+
+ /**
+ * The path to a file containing the password for
+ * the LDAP SSL truststore
+ */
+ public static final String LDAP_TRUSTSTORE_PASSWORD_FILE_KEY =
+ LDAP_TRUSTSTORE_PASSWORD_KEY + ".file";
+
/*
* User to bind to the LDAP server with
*/
@@ -226,6 +247,8 @@ public class LdapGroupsMapping
private boolean useSsl;
private String keystore;
private String keystorePass;
+ private String truststore;
+ private String truststorePass;
private String bindUser;
private String bindPassword;
private String userbaseDN;
@@ -526,8 +549,19 @@ public class LdapGroupsMapping
// Set up SSL security, if necessary
if (useSsl) {
env.put(Context.SECURITY_PROTOCOL, "ssl");
- System.setProperty("javax.net.ssl.keyStore", keystore);
- System.setProperty("javax.net.ssl.keyStorePassword", keystorePass);
+ if (!keystore.isEmpty()) {
+ System.setProperty("javax.net.ssl.keyStore", keystore);
+ }
+ if (!keystorePass.isEmpty()) {
+ System.setProperty("javax.net.ssl.keyStorePassword", keystorePass);
+ }
+ if (!truststore.isEmpty()) {
+ System.setProperty("javax.net.ssl.trustStore", truststore);
+ }
+ if (!truststorePass.isEmpty()) {
+ System.setProperty("javax.net.ssl.trustStorePassword",
+ truststorePass);
+ }
}
env.put(Context.SECURITY_PRINCIPAL, bindUser);
@@ -572,15 +606,10 @@ public class LdapGroupsMapping
if (ldapUrl == null || ldapUrl.isEmpty()) {
throw new RuntimeException("LDAP URL is not configured");
}
-
+
useSsl = conf.getBoolean(LDAP_USE_SSL_KEY, LDAP_USE_SSL_DEFAULT);
- keystore = conf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
-
- keystorePass = getPassword(conf, LDAP_KEYSTORE_PASSWORD_KEY,
- LDAP_KEYSTORE_PASSWORD_DEFAULT);
- if (keystorePass.isEmpty()) {
- keystorePass = extractPassword(conf.get(LDAP_KEYSTORE_PASSWORD_FILE_KEY,
- LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT));
+ if (useSsl) {
+ loadSslConf(conf);
}
bindUser = conf.get(BIND_USER_KEY, BIND_USER_DEFAULT);
@@ -643,6 +672,47 @@ public class LdapGroupsMapping
this.conf = conf;
}
+ private void loadSslConf(Configuration sslConf) {
+ keystore = sslConf.get(LDAP_KEYSTORE_KEY, LDAP_KEYSTORE_DEFAULT);
+ keystorePass = getPassword(sslConf, LDAP_KEYSTORE_PASSWORD_KEY,
+ LDAP_KEYSTORE_PASSWORD_DEFAULT);
+ if (keystorePass.isEmpty()) {
+ keystorePass = extractPassword(sslConf.get(
+ LDAP_KEYSTORE_PASSWORD_FILE_KEY,
+ LDAP_KEYSTORE_PASSWORD_FILE_DEFAULT));
+ }
+
+ truststore = sslConf.get(LDAP_TRUSTSTORE_KEY, "");
+ truststorePass = getPasswordFromCredentialProviders(
+ sslConf, LDAP_TRUSTSTORE_PASSWORD_KEY, "");
+ if (truststorePass.isEmpty()) {
+ truststorePass = extractPassword(
+ sslConf.get(LDAP_TRUSTSTORE_PASSWORD_FILE_KEY, ""));
+ }
+ }
+
+ String getPasswordFromCredentialProviders(
+ Configuration conf, String alias, String defaultPass) {
+ String password = defaultPass;
+ try {
+ char[] passchars = conf.getPasswordFromCredentialProviders(alias);
+ if (passchars != null) {
+ password = new String(passchars);
+ }
+ } catch (IOException ioe) {
+ LOG.warn("Exception while trying to get password for alias {}: {}",
+ alias, ioe);
+ }
+ return password;
+ }
+
+ /**
+ * Passwords should not be stored in configuration. Use
+ * {@link #getPasswordFromCredentialProviders(
+ * Configuration, String, String)}
+ * to avoid reading passwords from a configuration file.
+ */
+ @Deprecated
String getPassword(Configuration conf, String alias, String defaultPass) {
String password = defaultPass;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2216bde3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 49bfa28..ad24f56 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -317,6 +317,27 @@
</property>
<property>
+ <name>hadoop.security.group.mapping.ldap.ssl.truststore</name>
+ <value></value>
+ <description>
+ File path to the SSL truststore that contains the root certificate used to
+ sign the LDAP server's certificate. Specify this if the LDAP server's
+ certificate is not signed by a well known certificate authority.
+ </description>
+</property>
+
+<property>
+ <name>hadoop.security.group.mapping.ldap.ssl.truststore.password.file</name>
+ <value></value>
+ <description>
+ The path to a file containing the password of the LDAP SSL truststore.
+
+ IMPORTANT: This file should be readable only by the Unix user running
+ the daemons.
+ </description>
+</property>
+
+<property>
<name>hadoop.security.group.mapping.ldap.bind.user</name>
<value></value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2216bde3/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
index 806ed54..c6af930 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/GroupsMapping.md
@@ -112,6 +112,9 @@ For some LDAP servers, such as Active Directory, the user object returned in the
Therefore, it is possible to infer the user's groups from the first query without sending the second one, and it may reduce group name resolution latency incurred by the second query. If it fails to get group names, it will fall back to the typical two-query scenario and send the second query to get group names.
To enable this feature, set `hadoop.security.group.mapping.ldap.search.attr.memberof` to `memberOf`, and Hadoop will resolve group names using this attribute in the user object.
+If the LDAP server's certificate is not signed by a well known certificate authority, specify the path to the truststore in `hadoop.security.group.mapping.ldap.ssl.truststore`.
+Similar to keystore, specify the truststore password file in `hadoop.security.group.mapping.ldap.ssl.truststore.password.file`.
+
Composite Groups Mapping
--------
`CompositeGroupsMapping` works by enumerating a list of service providers in `hadoop.security.group.mapping.providers`.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: HDFS-13087. Snapshotted encryption
zone information should be immutable. Contributed by LiXin Ge.
Posted by na...@apache.org.
HDFS-13087. Snapshotted encryption zone information should be immutable. Contributed by LiXin Ge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2c6cfad5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2c6cfad5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2c6cfad5
Branch: refs/heads/YARN-3409
Commit: 2c6cfad5a31ca4d9126ecd2b3c43cca8543aacb4
Parents: e7e2019
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Mar 29 15:36:31 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Mar 29 15:46:34 2018 -0700
----------------------------------------------------------------------
.../server/namenode/EncryptionZoneManager.java | 74 +++++++++++++++++---
.../server/namenode/FSDirEncryptionZoneOp.java | 15 ++--
.../hdfs/server/namenode/FSDirXAttrOp.java | 8 ++-
.../hdfs/server/namenode/XAttrStorage.java | 9 ++-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 15 +++-
5 files changed, 97 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c6cfad5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 176ae1d..b1bca98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -33,6 +33,9 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
@@ -50,6 +53,7 @@ import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.security.AccessControlException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -107,6 +111,34 @@ public class EncryptionZoneManager {
String getKeyName() {
return keyName;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof EncryptionZoneInt)) {
+ return false;
+ }
+
+ EncryptionZoneInt b = (EncryptionZoneInt)o;
+ return new EqualsBuilder()
+ .append(inodeId, b.getINodeId())
+ .append(suite, b.getSuite())
+ .append(version, b.getVersion())
+ .append(keyName, b.getKeyName())
+ .isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+ return new HashCodeBuilder().
+ append(inodeId).
+ append(suite).
+ append(version).
+ append(keyName).
+ toHashCode();
+ }
}
private TreeMap<Long, EncryptionZoneInt> encryptionZones = null;
@@ -315,8 +347,8 @@ public class EncryptionZoneManager {
* <p/>
* Called while holding the FSDirectory lock.
*/
- boolean isInAnEZ(INodesInPath iip)
- throws UnresolvedLinkException, SnapshotAccessControlException {
+ boolean isInAnEZ(INodesInPath iip) throws UnresolvedLinkException,
+ SnapshotAccessControlException, IOException {
assert dir.hasReadLock();
return (getEncryptionZoneForPath(iip) != null);
}
@@ -341,7 +373,7 @@ public class EncryptionZoneManager {
* <p/>
* Called while holding the FSDirectory lock.
*/
- String getKeyName(final INodesInPath iip) {
+ String getKeyName(final INodesInPath iip) throws IOException {
assert dir.hasReadLock();
EncryptionZoneInt ezi = getEncryptionZoneForPath(iip);
if (ezi == null) {
@@ -356,19 +388,43 @@ public class EncryptionZoneManager {
* <p/>
* Called while holding the FSDirectory lock.
*/
- private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip) {
+ private EncryptionZoneInt getEncryptionZoneForPath(INodesInPath iip)
+ throws IOException{
assert dir.hasReadLock();
Preconditions.checkNotNull(iip);
if (!hasCreatedEncryptionZone()) {
return null;
}
+
+ int snapshotID = iip.getPathSnapshotId();
for (int i = iip.length() - 1; i >= 0; i--) {
final INode inode = iip.getINode(i);
- if (inode != null) {
+ if (inode == null || !inode.isDirectory()) {
+ //not found or not a directory, encryption zone is supported on
+ //directory only.
+ continue;
+ }
+ if (snapshotID == Snapshot.CURRENT_STATE_ID) {
final EncryptionZoneInt ezi = encryptionZones.get(inode.getId());
if (ezi != null) {
return ezi;
}
+ } else {
+ XAttr xAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName(
+ inode, snapshotID, CRYPTO_XATTR_ENCRYPTION_ZONE);
+ if (xAttr != null) {
+ try {
+ final HdfsProtos.ZoneEncryptionInfoProto ezProto =
+ HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xAttr.getValue());
+ return new EncryptionZoneInt(
+ inode.getId(), PBHelperClient.convert(ezProto.getSuite()),
+ PBHelperClient.convert(ezProto.getCryptoProtocolVersion()),
+ ezProto.getKeyName());
+ } catch (InvalidProtocolBufferException e) {
+ throw new IOException("Could not parse encryption zone for inode "
+ + iip.getPath(), e);
+ }
+ }
}
}
return null;
@@ -381,7 +437,8 @@ public class EncryptionZoneManager {
* <p/>
* Called while holding the FSDirectory lock.
*/
- private EncryptionZoneInt getParentEncryptionZoneForPath(INodesInPath iip) {
+ private EncryptionZoneInt getParentEncryptionZoneForPath(INodesInPath iip)
+ throws IOException {
assert dir.hasReadLock();
Preconditions.checkNotNull(iip);
INodesInPath parentIIP = iip.getParentINodesInPath();
@@ -395,7 +452,8 @@ public class EncryptionZoneManager {
* @param iip The INodesInPath of the path to check
* @return the EncryptionZone representing the ez for the path.
*/
- EncryptionZone getEZINodeForPath(INodesInPath iip) {
+ EncryptionZone getEZINodeForPath(INodesInPath iip)
+ throws IOException {
final EncryptionZoneInt ezi = getEncryptionZoneForPath(iip);
if (ezi == null) {
return null;
@@ -437,7 +495,7 @@ public class EncryptionZoneManager {
}
if (srcInEZ) {
- if (srcParentEZI != dstParentEZI) {
+ if (!srcParentEZI.equals(dstParentEZI)) {
final String srcEZPath = getFullPathName(srcParentEZI.getINodeId());
final String dstEZPath = getFullPathName(dstParentEZI.getINodeId());
final StringBuilder sb = new StringBuilder(srcIIP.getPath());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c6cfad5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index bf5652d..3d78172 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -205,7 +205,7 @@ final class FSDirEncryptionZoneOp {
}
static EncryptionZone getEZForPath(final FSDirectory fsd,
- final INodesInPath iip) {
+ final INodesInPath iip) throws IOException {
fsd.readLock();
try {
return fsd.ezManager.getEZINodeForPath(iip);
@@ -360,8 +360,9 @@ final class FSDirEncryptionZoneOp {
private static ZoneEncryptionInfoProto getZoneEncryptionInfoProto(
final INodesInPath iip) throws IOException {
- final XAttr fileXAttr = FSDirXAttrOp
- .unprotectedGetXAttrByPrefixedName(iip, CRYPTO_XATTR_ENCRYPTION_ZONE);
+ final XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName(
+ iip.getLastINode(), iip.getPathSnapshotId(),
+ CRYPTO_XATTR_ENCRYPTION_ZONE);
if (fileXAttr == null) {
throw new IOException(
"Could not find reencryption XAttr for file " + iip.getPath());
@@ -457,7 +458,8 @@ final class FSDirEncryptionZoneOp {
}
XAttr fileXAttr = FSDirXAttrOp.unprotectedGetXAttrByPrefixedName(
- iip, CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
+ iip.getLastINode(), iip.getPathSnapshotId(),
+ CRYPTO_XATTR_FILE_ENCRYPTION_INFO);
if (fileXAttr == null) {
NameNode.LOG.warn("Could not find encryption XAttr for file " +
iip.getPath() + " in encryption zone " + encryptionZone.getPath());
@@ -494,7 +496,7 @@ final class FSDirEncryptionZoneOp {
*/
static FileEncryptionInfo getFileEncryptionInfo(FSDirectory dir,
INodesInPath iip, EncryptionKeyInfo ezInfo)
- throws RetryStartFileException {
+ throws RetryStartFileException, IOException {
FileEncryptionInfo feInfo = null;
final EncryptionZone zone = getEZForPath(dir, iip);
if (zone != null) {
@@ -517,7 +519,8 @@ final class FSDirEncryptionZoneOp {
}
static boolean isInAnEZ(final FSDirectory fsd, final INodesInPath iip)
- throws UnresolvedLinkException, SnapshotAccessControlException {
+ throws UnresolvedLinkException, SnapshotAccessControlException,
+ IOException {
if (!fsd.ezManager.hasCreatedEncryptionZone()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c6cfad5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index 24a475f..9e95f90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -378,16 +378,18 @@ class FSDirXAttrOp {
String prefixedName) throws IOException {
fsd.readLock();
try {
- return XAttrStorage.readINodeXAttrByPrefixedName(iip, prefixedName);
+ return XAttrStorage.readINodeXAttrByPrefixedName(iip.getLastINode(),
+ iip.getPathSnapshotId(), prefixedName);
} finally {
fsd.readUnlock();
}
}
static XAttr unprotectedGetXAttrByPrefixedName(
- INodesInPath iip, String prefixedName)
+ INode inode, int snapshotId, String prefixedName)
throws IOException {
- return XAttrStorage.readINodeXAttrByPrefixedName(iip, prefixedName);
+ return XAttrStorage.readINodeXAttrByPrefixedName(
+ inode, snapshotId, prefixedName);
}
private static void checkXAttrChangeAccess(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c6cfad5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
index 8a91e2a..3b3747b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/XAttrStorage.java
@@ -47,14 +47,13 @@ public class XAttrStorage {
* <p/>
*
* @param inode INode to read
- * @param snapshotId
+ * @param snapshotId the snapshotId of the requested path
* @param prefixedName xAttr name with prefix
* @return the xAttr
*/
- public static XAttr readINodeXAttrByPrefixedName(INodesInPath iip,
- String prefixedName) {
- XAttrFeature f =
- iip.getLastINode().getXAttrFeature(iip.getPathSnapshotId());
+ public static XAttr readINodeXAttrByPrefixedName(INode inode, int snapshotId,
+ String prefixedName) {
+ XAttrFeature f = inode.getXAttrFeature(snapshotId);
return f == null ? null : f.getXAttr(prefixedName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2c6cfad5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index c541434..6f9ef29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -1413,11 +1413,20 @@ public class TestEncryptionZones {
fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
final Path snap2 = fs.createSnapshot(zoneParent, "snap2");
final Path snap2Zone = new Path(snap2, zone.getName());
+ assertEquals("Got unexpected ez path", zone.toString(),
+ dfsAdmin.getEncryptionZoneForPath(snap1Zone).getPath().toString());
assertNull("Expected null ez path",
dfsAdmin.getEncryptionZoneForPath(snap2Zone));
- // Create the encryption zone again
+ // Create the encryption zone again, and that shouldn't affect old snapshot
dfsAdmin.createEncryptionZone(zone, TEST_KEY2, NO_TRASH);
+ EncryptionZone ezSnap1 = dfsAdmin.getEncryptionZoneForPath(snap1Zone);
+ assertEquals("Got unexpected ez path", zone.toString(),
+ ezSnap1.getPath().toString());
+ assertEquals("Unexpected ez key", TEST_KEY, ezSnap1.getKeyName());
+ assertNull("Expected null ez path",
+ dfsAdmin.getEncryptionZoneForPath(snap2Zone));
+
final Path snap3 = fs.createSnapshot(zoneParent, "snap3");
final Path snap3Zone = new Path(snap3, zone.getName());
// Check that snap3's EZ has the correct settings
@@ -1426,10 +1435,12 @@ public class TestEncryptionZones {
ezSnap3.getPath().toString());
assertEquals("Unexpected ez key", TEST_KEY2, ezSnap3.getKeyName());
// Check that older snapshots still have the old EZ settings
- EncryptionZone ezSnap1 = dfsAdmin.getEncryptionZoneForPath(snap1Zone);
+ ezSnap1 = dfsAdmin.getEncryptionZoneForPath(snap1Zone);
assertEquals("Got unexpected ez path", zone.toString(),
ezSnap1.getPath().toString());
assertEquals("Unexpected ez key", TEST_KEY, ezSnap1.getKeyName());
+ assertNull("Expected null ez path",
+ dfsAdmin.getEncryptionZoneForPath(snap2Zone));
// Check that listEZs only shows the current filesystem state
ArrayList<EncryptionZone> listZones = Lists.newArrayList();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-8086. ManagedParentQueue with
no leaf queues cause JS error in new UI. (Suma Shivaprasad via wangda)
Posted by na...@apache.org.
YARN-8086. ManagedParentQueue with no leaf queues cause JS error in new UI. (Suma Shivaprasad via wangda)
Change-Id: I6d82af91adec02698afddde6883b1fe1924e6041
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e7e20190
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e7e20190
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e7e20190
Branch: refs/heads/YARN-3409
Commit: e7e20190dface4117abe70dcfd96d19b0e4f5339
Parents: 8fb00c3
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Mar 29 15:03:31 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Mar 29 15:03:31 2018 -0700
----------------------------------------------------------------------
.../src/main/webapp/app/serializers/yarn-queue/capacity-queue.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e7e20190/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
index e838255..57714e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -24,7 +24,7 @@ export default DS.JSONAPISerializer.extend({
normalizeSingleResponse(store, primaryModelClass, payload, id,
requestType) {
var children = [];
- if (payload.queues) {
+ if (payload.queues && payload.queues.queue) {
payload.queues.queue.forEach(function(queue) {
children.push(queue.queueName);
});
@@ -122,7 +122,7 @@ export default DS.JSONAPISerializer.extend({
data.push(result.queue);
includedData = includedData.concat(result.includedData);
- if (payload.queues) {
+ if (payload.queues && payload.queues.queue) {
for (var i = 0; i < payload.queues.queue.length; i++) {
var queue = payload.queues.queue[i];
queue.myParent = payload.queueName;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-8076. Support to specify
application tags in distributed shell. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-8076. Support to specify application tags in distributed shell. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/431076f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/431076f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/431076f6
Branch: refs/heads/YARN-3409
Commit: 431076f63751f855ab6036ff85825a8552257b93
Parents: 3d185d6
Author: Sunil G <su...@apache.org>
Authored: Thu Mar 29 15:55:39 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Mar 29 15:55:39 2018 +0530
----------------------------------------------------------------------
.../yarn/applications/distributedshell/Client.java | 15 +++++++++++++++
1 file changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/431076f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index d6a753a..61879d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -230,6 +230,9 @@ public class Client {
// Docker client configuration
private String dockerClientConfig = null;
+ // Application tags
+ private Set<String> applicationTags = new HashSet<>();
+
// Command line options
private Options opts;
@@ -384,6 +387,7 @@ public class Client {
"Placement specification. Please note, if this option is specified,"
+ " The \"num_containers\" option will be ignored. All requested"
+ " containers will be of type GUARANTEED" );
+ opts.addOption("application_tags", true, "Application tags.");
}
/**
@@ -604,6 +608,14 @@ public class Client {
if (cliParser.hasOption("docker_client_config")) {
dockerClientConfig = cliParser.getOptionValue("docker_client_config");
}
+
+ if (cliParser.hasOption("application_tags")) {
+ String applicationTagsStr = cliParser.getOptionValue("application_tags");
+ String[] appTags = applicationTagsStr.split(",");
+ for (String appTag : appTags) {
+ this.applicationTags.add(appTag.trim());
+ }
+ }
return true;
}
@@ -729,6 +741,9 @@ public class Client {
}
Set<String> tags = new HashSet<String>();
+ if (applicationTags != null) {
+ tags.addAll(applicationTags);
+ }
if (flowName != null) {
tags.add(TimelineUtils.generateFlowNameTag(flowName));
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-8072. RM log is getting
flooded with MemoryPlacementConstraintManager info logs. (Zian Chen via
wangda)
Posted by na...@apache.org.
YARN-8072. RM log is getting flooded with MemoryPlacementConstraintManager info logs. (Zian Chen via wangda)
Change-Id: I2567f4669ad868e13c4301fe44c4bde85bb92c37
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/24bc5e0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/24bc5e0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/24bc5e0e
Branch: refs/heads/YARN-3409
Commit: 24bc5e0e4d919fddae983fc13d4ba4e4e5cb9533
Parents: 9eef19b
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Mar 26 11:20:16 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Mar 26 11:20:16 2018 -0700
----------------------------------------------------------------------
.../constraint/MemoryPlacementConstraintManager.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/24bc5e0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
index 5cb8b99..bf04672 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/constraint/MemoryPlacementConstraintManager.java
@@ -184,8 +184,10 @@ public class MemoryPlacementConstraintManager
try {
readLock.lock();
if (appConstraints.get(appId) == null) {
- LOG.info("Application {} is not registered in the Placement "
- + "Constraint Manager.", appId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Application {} is not registered in the Placement "
+ + "Constraint Manager.", appId);
+ }
return null;
}
@@ -213,8 +215,10 @@ public class MemoryPlacementConstraintManager
try {
readLock.lock();
if (appConstraints.get(appId) == null) {
- LOG.info("Application {} is not registered in the Placement "
- + "Constraint Manager.", appId);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Application {} is not registered in the Placement "
+ + "Constraint Manager.", appId);
+ }
return null;
}
// TODO: Merge this constraint with the global one for this tag, if one
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-7856. Validate Node
Attributes from NM. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-7856. Validate Node Attributes from NM. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14a92509
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14a92509
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14a92509
Branch: refs/heads/YARN-3409
Commit: 14a9250912a75bb8964da8c260286e3763edc666
Parents: 06fbea4
Author: Sunil G <su...@apache.org>
Authored: Tue Feb 27 08:15:42 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 11:30:55 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/NodeAttribute.java | 2 ++
.../hadoop/yarn/nodelabels/NodeLabelUtil.java | 31 ++++++++++++++++++++
.../ScriptBasedNodeAttributesProvider.java | 25 ++++++++++++++--
.../TestScriptBasedNodeAttributesProvider.java | 27 +++++++++++++++++
4 files changed, 83 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 01c70b2..4f6846b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -46,6 +46,8 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class NodeAttribute {
public static final String DEFAULT_PREFIX = "";
+ public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
+ public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
public static NodeAttribute newInstance(String attributeName,
NodeAttributeType attributeType, String attributeValue) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
index d918712..fdfd0ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -17,7 +17,11 @@
*/
package org.apache.hadoop.yarn.nodelabels;
+import com.google.common.base.Strings;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
import java.io.IOException;
+import java.util.Set;
import java.util.regex.Pattern;
/**
@@ -94,4 +98,31 @@ public final class NodeLabelUtil {
+ ", now it is= " + prefix);
}
}
+
+ /**
+ * Validate if a given set of attributes are valid. Attributes could be
+ * invalid if any of following conditions is met:
+ *
+ * <ul>
+ * <li>Missing prefix: the attribute doesn't have prefix defined</li>
+ * <li>Malformed attribute prefix: the prefix is not in valid format</li>
+ * </ul>
+ * @param attributeSet
+ * @throws IOException
+ */
+ public static void validateNodeAttributes(Set<NodeAttribute> attributeSet)
+ throws IOException {
+ if (attributeSet != null && !attributeSet.isEmpty()) {
+ for (NodeAttribute nodeAttribute : attributeSet) {
+ String prefix = nodeAttribute.getAttributePrefix();
+ if (Strings.isNullOrEmpty(prefix)) {
+ throw new IOException("Attribute prefix must be set");
+ }
+ // Verify attribute prefix format.
+ checkAndThrowAttributePrefix(prefix);
+ // Verify attribute name format.
+ checkAndThrowLabelName(nodeAttribute.getAttributeName());
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
index 06771ba..4621434 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeAttributesProvider.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
import java.io.IOException;
import java.util.HashSet;
@@ -116,13 +117,33 @@ public class ScriptBasedNodeAttributesProvider extends NodeAttributesProvider{
+ NODE_ATTRIBUTE_DELIMITER + "ATTRIBUTE_VALUE; but get "
+ nodeAttribute);
}
+ // Automatically setup prefix for collected attributes
NodeAttribute na = NodeAttribute
- .newInstance(attributeStrs[0],
+ .newInstance(NodeAttribute.PREFIX_DISTRIBUTED,
+ attributeStrs[0],
NodeAttributeType.valueOf(attributeStrs[1]),
attributeStrs[2]);
- attributeSet.add(na);
+
+ // Since a NodeAttribute is identical with another one as long as
+ // their prefix and name are same, to avoid attributes getting
+ // overwritten by ambiguous attribute, make sure it fails in such
+ // case.
+ if (!attributeSet.add(na)) {
+ throw new IOException("Ambiguous node attribute is found: "
+ + na.toString() + ", a same attribute already exists");
+ }
}
}
+
+ // Before updating the attributes to the provider,
+ // verify if they are valid
+ try {
+ NodeLabelUtil.validateNodeAttributes(attributeSet);
+ } catch (IOException e) {
+ throw new IOException("Node attributes collected by the script "
+ + "contains some invalidate entries. Detail message: "
+ + e.getMessage());
+ }
return attributeSet;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14a92509/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
index 58d2d20..f764626 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
@@ -220,4 +220,31 @@ public class TestScriptBasedNodeAttributesProvider {
}
}, 500, 3000);
}
+
+ @Test
+ public void testNodeAttributesValidation() throws Exception{
+ // Script output contains ambiguous node attributes
+ String scriptContent = "echo NODE_ATTRIBUTE:host,STRING,host1234\n "
+ + "echo NODE_ATTRIBUTE:host,STRING,host2345\n "
+ + "echo NODE_ATTRIBUTE:ip,STRING,10.0.0.1";
+
+ writeNodeAttributeScriptFile(scriptContent, true);
+
+ nodeAttributesProvider.init(getConfForNodeAttributeScript());
+ nodeAttributesProvider.start();
+
+ // There should be no attributes found, and we should
+ // see Malformed output warnings in the log
+ try {
+ GenericTestUtils
+ .waitFor(() -> nodeAttributesProvider
+ .getDescriptors().size() == 3,
+ 500, 3000);
+ Assert.fail("This test should timeout because the provide is unable"
+ + " to parse any attributes from the script output.");
+ } catch (TimeoutException e) {
+ Assert.assertEquals(0, nodeAttributesProvider
+ .getDescriptors().size());
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-13352. RBF: Add xsl
stylesheet for hdfs-rbf-default.xml. Contributed by Takanobu Asanuma.
Posted by na...@apache.org.
HDFS-13352. RBF: Add xsl stylesheet for hdfs-rbf-default.xml. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cef0114
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cef0114
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cef0114
Branch: refs/heads/YARN-3409
Commit: 4cef01147f98451de8acaa90f42a4d4cd4d0af45
Parents: c22d62b
Author: Inigo Goiri <in...@apache.org>
Authored: Tue Mar 27 10:31:03 2018 -0700
Committer: Inigo Goiri <in...@apache.org>
Committed: Tue Mar 27 10:31:03 2018 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml | 1 +
.../src/main/xsl/configuration.xsl | 37 ++++++++++++++++++++
2 files changed, 38 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cef0114/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
index 9683744..b883726 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/pom.xml
@@ -167,6 +167,7 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<configuration>
<tasks>
<copy file="src/main/resources/hdfs-rbf-default.xml" todir="src/site/resources"/>
+ <copy file="src/main/xsl/configuration.xsl" todir="src/site/resources"/>
</tasks>
</configuration>
</execution>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cef0114/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl
new file mode 100644
index 0000000..d23168f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/xsl/configuration.xsl
@@ -0,0 +1,37 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
+<xsl:output method="html"/>
+<xsl:template match="configuration">
+<html>
+<body>
+<table border="1">
+<tr>
+ <td>name</td>
+ <td>value</td>
+ <td>description</td>
+</tr>
+<xsl:for-each select="property">
+<tr>
+ <td><a name="{name}"><xsl:value-of select="name"/></a></td>
+ <td><xsl:value-of select="value"/></td>
+ <td><xsl:value-of select="description"/></td>
+</tr>
+</xsl:for-each>
+</table>
+</body>
+</html>
+</xsl:template>
+</xsl:stylesheet>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-8033. CLI Integration with
NodeAttributesManagerImpl. Contributed by Naganarasimha G R.
Posted by na...@apache.org.
YARN-8033. CLI Integration with NodeAttributesManagerImpl. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c20126ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c20126ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c20126ac
Branch: refs/heads/YARN-3409
Commit: c20126acfbc82adce4801f20b764e929d1686c1f
Parents: 4724556
Author: bibinchundatt <bi...@apache.org>
Authored: Sun Apr 1 19:24:00 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 14:11:12 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/api/records/NodeAttribute.java | 3 +-
.../yarn/client/cli/NodeAttributesCLI.java | 18 +-
.../yarn/client/cli/TestNodeAttributesCLI.java | 18 ++
.../server/resourcemanager/AdminService.java | 97 +++++++++++
.../resourcemanager/TestRMAdminService.java | 170 +++++++++++++++++--
5 files changed, 282 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
index 4f6846b..25ac9ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeAttribute.java
@@ -45,13 +45,12 @@ import org.apache.hadoop.yarn.util.Records;
@Unstable
public abstract class NodeAttribute {
- public static final String DEFAULT_PREFIX = "";
public static final String PREFIX_DISTRIBUTED = "nm.yarn.io";
public static final String PREFIX_CENTRALIZED = "rm.yarn.io";
public static NodeAttribute newInstance(String attributeName,
NodeAttributeType attributeType, String attributeValue) {
- return newInstance(DEFAULT_PREFIX, attributeName, attributeType,
+ return newInstance(PREFIX_CENTRALIZED, attributeName, attributeType,
attributeValue);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
index 2eff155..df5a57d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeAttributesCLI.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.client.cli;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -50,6 +52,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappin
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Lists;
/**
* CLI to map attributes to Nodes.
@@ -311,7 +314,7 @@ public class NodeAttributesCLI extends Configured implements Tool {
*/
private List<NodeToAttributes> buildNodeLabelsMapFromStr(String args,
boolean validateForAttributes, AttributeMappingOperationType operation) {
- List<NodeToAttributes> nodeToAttributesList = new ArrayList<>();
+ Map<String,NodeToAttributes> nodeToAttributesMap = new HashMap<>();
for (String nodeToAttributesStr : args.split("[ \n]")) {
// for each node to attribute mapping
nodeToAttributesStr = nodeToAttributesStr.trim();
@@ -384,8 +387,9 @@ public class NodeAttributesCLI extends Configured implements Tool {
// TODO when we support different type of attribute type we need to
// cross verify whether input attributes itself is not violating
// attribute Name to Type mapping.
- attributesList.add(NodeAttribute.newInstance(attributeName.trim(),
- attributeType, attributeValue.trim()));
+ attributesList
+ .add(NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED,
+ attributeName.trim(), attributeType, attributeValue.trim()));
}
}
if (validateForAttributes) {
@@ -393,14 +397,14 @@ public class NodeAttributesCLI extends Configured implements Tool {
"Attributes cannot be null or empty for Operation "
+ operation.name() + " on the node " + node);
}
- nodeToAttributesList
- .add(NodeToAttributes.newInstance(node, attributesList));
+ nodeToAttributesMap
+ .put(node,NodeToAttributes.newInstance(node, attributesList));
}
- if (nodeToAttributesList.isEmpty()) {
+ if (nodeToAttributesMap.isEmpty()) {
throw new IllegalArgumentException(NO_MAPPING_ERR_MSG);
}
- return nodeToAttributesList;
+ return Lists.newArrayList(nodeToAttributesMap.values());
}
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
index cc92a93..bbd5ca3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestNodeAttributesCLI.java
@@ -301,6 +301,24 @@ public class TestNodeAttributesCLI {
NodesToAttributesMappingRequest.newInstance(
AttributeMappingOperationType.ADD, nodeAttributesList, true);
assertTrue(request.equals(expected));
+
+ // --------------------------------
+ // with Duplicate mappings for a host
+ // --------------------------------
+ args = new String[] { "-add", "x:key2=123,key3=abc x:key4(string)",
+ "-failOnUnknownNodes" };
+ assertTrue("Should not fail as attribute has been properly mapped",
+ 0 == runTool(args));
+ nodeAttributesList = new ArrayList<>();
+ attributes = new ArrayList<>();
+ attributes
+ .add(NodeAttribute.newInstance("key4", NodeAttributeType.STRING, ""));
+ nodeAttributesList.add(NodeToAttributes.newInstance("x", attributes));
+
+ expected =
+ NodesToAttributesMappingRequest.newInstance(
+ AttributeMappingOperationType.ADD, nodeAttributesList, true);
+ assertTrue(request.equals(expected));
}
private void assertFailureMessageContains(String... messages) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index bff3f2c..12f8aaf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -23,6 +23,8 @@ import java.io.InputStream;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -50,6 +52,7 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.conf.HAUtil;
@@ -59,6 +62,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
@@ -66,6 +70,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsR
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.CheckForDecommissioningNodesResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
@@ -968,7 +973,99 @@ public class AdminService extends CompositeService implements
public NodesToAttributesMappingResponse mapAttributesToNodes(
NodesToAttributesMappingRequest request)
throws YarnException, IOException {
+
+ final String operation = "mapAttributesToNodes";
+ final String msg = "Map Attributes to Nodes";
+ UserGroupInformation user = checkAcls(operation);
+ checkRMStatus(user.getShortUserName(), operation, msg);
+
+
+ List<NodeToAttributes> nodesToAttributes = request.getNodesToAttributes();
+ boolean failOnUnknownNodes = request.getFailOnUnknownNodes();
+
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping =
+ validateAndFetch(nodesToAttributes, failOnUnknownNodes);
+
+ NodeAttributesManager nodeAttributesManager =
+ rm.getRMContext().getNodeAttributesManager();
+ try {
+ switch (request.getOperation()) {
+ case ADD:
+ nodeAttributesManager.addNodeAttributes(nodeAttributeMapping);
+ break;
+ case REMOVE:
+ nodeAttributesManager.removeNodeAttributes(nodeAttributeMapping);
+ break;
+ case REPLACE:
+ nodeAttributesManager.replaceNodeAttributes(
+ NodeAttribute.PREFIX_CENTRALIZED, nodeAttributeMapping);
+ break;
+ default:
+ throw new IOException("Invalid operation " + request.getOperation()
+ + " specified in the mapAttributesToNodes request ");
+
+ }
+ } catch (IOException ioe) {
+ throw logAndWrapException(ioe, user.getShortUserName(), operation, msg);
+ }
+ RMAuditLogger.logSuccess(user.getShortUserName(), operation,
+ "AdminService");
return recordFactory
.newRecordInstance(NodesToAttributesMappingResponse.class);
}
+
+ /**
+ * @param nodesToAttributesMapping input to be validated
+ * @param failOnUnknownNodes indicates to fail if the nodes are not available.
+ * @return the map of Node host name to set of NodeAttributes
+ * @throws IOException if validation fails for node existence or the attribute
+ * has a wrong prefix
+ */
+ private Map<String, Set<NodeAttribute>> validateAndFetch(
+ List<NodeToAttributes> nodesToAttributesMapping,
+ boolean failOnUnknownNodes) throws IOException {
+ Map<String, Set<NodeAttribute>> attributeMapping = new HashMap<>();
+ List<String> invalidNodes = new ArrayList<>();
+ for (NodeToAttributes nodeToAttributes : nodesToAttributesMapping) {
+ String node = nodeToAttributes.getNode();
+ if (!validateForInvalidNode(node, failOnUnknownNodes)) {
+ invalidNodes.add(node);
+ continue;
+ }
+ List<NodeAttribute> nodeAttributes = nodeToAttributes.getNodeAttributes();
+ if (!nodeAttributes.stream()
+ .allMatch(nodeAttribute -> NodeAttribute.PREFIX_CENTRALIZED
+ .equals(nodeAttribute.getAttributePrefix()))) {
+ throw new IOException("Invalid Attribute Mapping for the node " + node
+ + ". Prefix should be " + NodeAttribute.PREFIX_CENTRALIZED);
+ }
+ attributeMapping.put(node, new HashSet<>(nodeAttributes));
+ }
+ if (!invalidNodes.isEmpty()) {
+ String message = " Following nodes does not exist : " + invalidNodes;
+ LOG.error(message);
+ throw new IOException(message);
+ }
+ return attributeMapping;
+ }
+
+ /**
+ * @param node
+ * @return true if valid else false;
+ */
+ private boolean validateForInvalidNode(String node,
+ boolean failOnUnknownNodes) {
+ if (!failOnUnknownNodes) {
+ return true;
+ }
+ // both active and inactive nodes are recognized as known nodes
+ boolean isKnown = rm.getRMContext().getRMNodes().keySet().stream()
+ .anyMatch(activeNode -> activeNode.getHost().equals(node));
+
+ if (!isKnown) {
+ isKnown = rm.getRMContext().getInactiveRMNodes().keySet().stream()
+ .anyMatch(inactiveNode -> inactiveNode.getHost().equals(node));
+ }
+ return isKnown;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c20126ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index 03fc081..523ba1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -53,6 +53,8 @@ import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.api.records.DecommissionType;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -60,6 +62,9 @@ import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshAdminAclsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshClusterMaxPriorityRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RefreshNodesRequest;
@@ -85,11 +90,14 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
import static org.junit.Assert.assertTrue;
@@ -1201,21 +1209,7 @@ public class TestRMAdminService {
((RMContextImpl) rm.getRMContext())
.setHAServiceState(HAServiceState.ACTIVE);
- Map<NodeId, RMNode> rmNodes = rm.getRMContext().getRMNodes();
- rmNodes.put(NodeId.newInstance("host1", 1111),
- new RMNodeImpl(null, rm.getRMContext(), "host1", 0, 0, null, null,
- null));
- rmNodes.put(NodeId.newInstance("host2", 2222),
- new RMNodeImpl(null, rm.getRMContext(), "host2", 0, 0, null, null,
- null));
- rmNodes.put(NodeId.newInstance("host3", 3333),
- new RMNodeImpl(null, rm.getRMContext(), "host3", 0, 0, null, null,
- null));
- Map<NodeId, RMNode> rmInactiveNodes = rm.getRMContext()
- .getInactiveRMNodes();
- rmInactiveNodes.put(NodeId.newInstance("host4", 4444),
- new RMNodeImpl(null, rm.getRMContext(), "host4", 0, 0, null, null,
- null));
+ setActiveAndInactiveNodes(rm);
RMNodeLabelsManager labelMgr = rm.rmContext.getNodeLabelManager();
// by default, distributed configuration for node label is disabled, this
@@ -1550,4 +1544,150 @@ public class TestRMAdminService {
Assert.assertTrue(
response.getNodeLabelList().containsAll(Arrays.asList(labelX, labelY)));
}
+
+ @Test(timeout = 30000)
+ public void testMapAttributesToNodes() throws Exception, YarnException {
+ // 1. Need to test for the Invalid Node
+ // 1.1. Need to test for active nodes
+ // 1.2. Need to test for Inactive nodes
+ // 1.3. Test with Single Node invalid
+ // 1.4. Need to test with port (should fail)
+ // 1.5. Test with unknown node when failOnUnknownNodes is false
+
+ // also test : 3. Ensure Appropriate manager Method call is done
+ rm = new MockRM();
+
+ NodeAttributesManager spiedAttributesManager =
+ Mockito.spy(rm.getRMContext().getNodeAttributesManager());
+ rm.getRMContext().setNodeAttributesManager(spiedAttributesManager);
+
+ ((RMContextImpl) rm.getRMContext())
+ .setHAServiceState(HAServiceState.ACTIVE);
+ setActiveAndInactiveNodes(rm);
+ // by default, distributed configuration for node label is disabled, this
+ // should pass
+ NodesToAttributesMappingRequest request =
+ NodesToAttributesMappingRequest
+ .newInstance(AttributeMappingOperationType.ADD,
+ ImmutableList.of(NodeToAttributes.newInstance("host1",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_CENTRALIZED, "x",
+ NodeAttributeType.STRING, "dfasdf")))),
+ true);
+ try {
+ rm.adminService.mapAttributesToNodes(request);
+ } catch (Exception ex) {
+ fail("should not fail on known node in active state" + ex.getMessage());
+ }
+ Mockito.verify(spiedAttributesManager, Mockito.times(1))
+ .addNodeAttributes(Mockito.anyMap());
+
+ request =
+ NodesToAttributesMappingRequest
+ .newInstance(AttributeMappingOperationType.REMOVE,
+ ImmutableList.of(NodeToAttributes.newInstance("host4",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_CENTRALIZED, "x",
+ NodeAttributeType.STRING, "dfasdf")))),
+ true);
+ try {
+ rm.adminService.mapAttributesToNodes(request);
+ } catch (Exception ex) {
+ fail("should not fail on known node in inactive state" + ex.getMessage());
+ }
+ Mockito.verify(spiedAttributesManager, Mockito.times(1))
+ .removeNodeAttributes(Mockito.anyMap());
+
+ request =
+ NodesToAttributesMappingRequest
+ .newInstance(AttributeMappingOperationType.ADD,
+ ImmutableList.of(NodeToAttributes.newInstance("host5",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_CENTRALIZED, "x",
+ NodeAttributeType.STRING, "dfasdf")))),
+ true);
+ try {
+ rm.adminService.mapAttributesToNodes(request);
+ fail("host5 is not a valid node, It should have failed");
+ } catch (Exception ex) {
+ Assert.assertEquals("Exception Message is not as desired",
+ " Following nodes does not exist : [host5]", ex.getMessage());
+ }
+
+ request =
+ NodesToAttributesMappingRequest
+ .newInstance(AttributeMappingOperationType.ADD, ImmutableList.of(
+ NodeToAttributes.newInstance("host4:8889",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_CENTRALIZED, "x",
+ NodeAttributeType.STRING, "dfasdf"))),
+ NodeToAttributes.newInstance("host2:8889",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_CENTRALIZED, "x",
+ NodeAttributeType.STRING, "dfasdf")))),
+ true);
+ try {
+ // port if added in CLI it fails in the client itself. Here we just check
+ // against hostname hence the message as : nodes does not exist.
+ rm.adminService.mapAttributesToNodes(request);
+ fail("host with the port should fail as only hostnames are validated");
+ } catch (Exception ex) {
+ Assert.assertEquals("Exception Message is not as desired",
+ " Following nodes does not exist : [host4:8889, host2:8889]",
+ ex.getMessage());
+ }
+
+ request =
+ NodesToAttributesMappingRequest
+ .newInstance(AttributeMappingOperationType.REPLACE,
+ ImmutableList.of(NodeToAttributes.newInstance("host5",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_CENTRALIZED, "x",
+ NodeAttributeType.STRING, "dfasdf")))),
+ false);
+ try {
+ rm.adminService.mapAttributesToNodes(request);
+ } catch (Exception ex) {
+ fail("This operation should not fail as failOnUnknownNodes is false : "
+ + ex.getMessage());
+ }
+ Mockito.verify(spiedAttributesManager, Mockito.times(1))
+ .replaceNodeAttributes(Mockito.eq(NodeAttribute.PREFIX_CENTRALIZED),
+ Mockito.anyMap());
+
+ // 2. fail on invalid prefix
+ request =
+ NodesToAttributesMappingRequest
+ .newInstance(AttributeMappingOperationType.ADD,
+ ImmutableList.of(NodeToAttributes.newInstance("host5",
+ ImmutableList.of(NodeAttribute.newInstance(
+ NodeAttribute.PREFIX_DISTRIBUTED, "x",
+ NodeAttributeType.STRING, "dfasdf")))),
+ false);
+ try {
+ rm.adminService.mapAttributesToNodes(request);
+ fail("This operation should fail as prefix should be \"nm.yarn.io\".");
+ } catch (Exception ex) {
+ Assert.assertEquals("Exception Message is not as desired",
+ "Invalid Attribute Mapping for the node host5. Prefix should be "
+ + "rm.yarn.io",
+ ex.getMessage());
+ }
+
+ rm.close();
+ }
+
+ private void setActiveAndInactiveNodes(ResourceManager resourceManager) {
+ Map<NodeId, RMNode> rmNodes = resourceManager.getRMContext().getRMNodes();
+ rmNodes.put(NodeId.newInstance("host1", 1111), new RMNodeImpl(null,
+ resourceManager.getRMContext(), "host1", 0, 0, null, null, null));
+ rmNodes.put(NodeId.newInstance("host2", 2222), new RMNodeImpl(null,
+ resourceManager.getRMContext(), "host2", 0, 0, null, null, null));
+ rmNodes.put(NodeId.newInstance("host3", 3333), new RMNodeImpl(null,
+ resourceManager.getRMContext(), "host3", 0, 0, null, null, null));
+ Map<NodeId, RMNode> rmInactiveNodes =
+ resourceManager.getRMContext().getInactiveRMNodes();
+ rmInactiveNodes.put(NodeId.newInstance("host4", 4444), new RMNodeImpl(null,
+ resourceManager.getRMContext(), "host4", 0, 0, null, null, null));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-8069. Clean up example
hostnames for RegistryDNS. Contributed by Billie Rinaldi
Posted by na...@apache.org.
YARN-8069. Clean up example hostnames for RegistryDNS.
Contributed by Billie Rinaldi
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d185d62
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d185d62
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d185d62
Branch: refs/heads/YARN-3409
Commit: 3d185d62fc392dd5313f8e36b9674b9200638463
Parents: a991e89
Author: Eric Yang <ey...@apache.org>
Authored: Wed Mar 28 20:17:37 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Wed Mar 28 20:17:37 2018 -0400
----------------------------------------------------------------------
.../registry/server/dns/TestRegistryDNS.java | 66 ++++++++++----------
.../test/resources/0.17.172.in-addr.arpa.zone | 24 +++----
.../markdown/yarn-service/Configurations.md | 4 +-
3 files changed, 47 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d185d62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
index 9c369d0..bce73ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/server/dns/TestRegistryDNS.java
@@ -156,7 +156,7 @@ public class TestRegistryDNS extends Assert {
protected Configuration createConfiguration() {
Configuration conf = new Configuration();
- conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test");
conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0");
conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
return conf;
@@ -179,39 +179,39 @@ public class TestRegistryDNS extends Assert {
"/registry/users/root/services/org-apache-slider/test1/", record);
// start assessing whether correct records are available
- Record[] recs = assertDNSQuery("test1.root.hwx.test.");
+ Record[] recs = assertDNSQuery("test1.root.dev.test.");
assertEquals("wrong result", "192.168.1.5",
((ARecord) recs[0]).getAddress().getHostAddress());
- recs = assertDNSQuery("management-api.test1.root.hwx.test.", 2);
- assertEquals("wrong target name", "test1.root.hwx.test.",
+ recs = assertDNSQuery("management-api.test1.root.dev.test.", 2);
+ assertEquals("wrong target name", "test1.root.dev.test.",
((CNAMERecord) recs[0]).getTarget().toString());
assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
- recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.",
+ recs = assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.",
Type.SRV, 1);
assertTrue("not an SRV record", recs[0] instanceof SRVRecord);
assertEquals("wrong port", 1026, ((SRVRecord) recs[0]).getPort());
- recs = assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", 2);
- assertEquals("wrong target name", "test1.root.hwx.test.",
+ recs = assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", 2);
+ assertEquals("wrong target name", "test1.root.dev.test.",
((CNAMERecord) recs[0]).getTarget().toString());
assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
- recs = assertDNSQuery("http-api.test1.root.hwx.test.", 2);
- assertEquals("wrong target name", "test1.root.hwx.test.",
+ recs = assertDNSQuery("http-api.test1.root.dev.test.", 2);
+ assertEquals("wrong target name", "test1.root.dev.test.",
((CNAMERecord) recs[0]).getTarget().toString());
assertTrue("not an ARecord", recs[isSecure() ? 2 : 1] instanceof ARecord);
- recs = assertDNSQuery("http-api.test1.root.hwx.test.", Type.SRV,
+ recs = assertDNSQuery("http-api.test1.root.dev.test.", Type.SRV,
1);
assertTrue("not an SRV record", recs[0] instanceof SRVRecord);
assertEquals("wrong port", 1027, ((SRVRecord) recs[0]).getPort());
- assertDNSQuery("test1.root.hwx.test.", Type.TXT, 3);
- assertDNSQuery("appmaster-ipc-api.test1.root.hwx.test.", Type.TXT, 1);
- assertDNSQuery("http-api.test1.root.hwx.test.", Type.TXT, 1);
- assertDNSQuery("management-api.test1.root.hwx.test.", Type.TXT, 1);
+ assertDNSQuery("test1.root.dev.test.", Type.TXT, 3);
+ assertDNSQuery("appmaster-ipc-api.test1.root.dev.test.", Type.TXT, 1);
+ assertDNSQuery("http-api.test1.root.dev.test.", Type.TXT, 1);
+ assertDNSQuery("management-api.test1.root.dev.test.", Type.TXT, 1);
}
@Test
@@ -225,11 +225,11 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Record[] recs =
- assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.dev.test.");
assertEquals("wrong result", "172.17.0.19",
((ARecord) recs[0]).getAddress().getHostAddress());
- recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1);
+ recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1);
assertTrue("not an ARecord", recs[0] instanceof ARecord);
}
@@ -243,7 +243,7 @@ public class TestRegistryDNS extends Assert {
record);
Name name =
- Name.fromString("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ Name.fromString("ctr-e50-1451931954322-0016-01-000002.dev.test.");
Record question = Record.newRecord(name, Type.A, DClass.IN);
Message query = Message.newQuery(question);
byte[] responseBytes = registryDNS.generateReply(query, null);
@@ -263,12 +263,12 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Record[] recs = assertDNSQuery(
- "ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ "ctr-e50-1451931954322-0016-01-000002.dev.test.");
assertEquals("wrong result", "172.17.0.19",
((ARecord) recs[0]).getAddress().getHostAddress());
assertEquals("wrong ttl", 30L, recs[0].getTTL());
- recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1);
+ recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1);
assertTrue("not an ARecord", recs[0] instanceof ARecord);
assertEquals("wrong ttl", 30L, recs[0].getTTL());
@@ -286,7 +286,7 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
assertEquals("wrong result",
- "comp-name.test1.root.hwx.test.",
+ "comp-name.test1.root.dev.test.",
((PTRRecord) recs[0]).getTarget().toString());
}
@@ -294,7 +294,7 @@ public class TestRegistryDNS extends Assert {
public void testReverseLookupInLargeNetwork() throws Exception {
setRegistryDNS(new RegistryDNS("TestRegistry"));
Configuration conf = createConfiguration();
- conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test");
conf.set(KEY_DNS_ZONE_SUBNET, "172.17.0.0");
conf.set(KEY_DNS_ZONE_MASK, "255.255.224.0");
conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
@@ -312,7 +312,7 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Record[] recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
assertEquals("wrong result",
- "comp-name.test1.root.hwx.test.",
+ "comp-name.test1.root.dev.test.",
((PTRRecord) recs[0]).getTarget().toString());
}
@@ -348,7 +348,7 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Name name =
- Name.fromString("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ Name.fromString("ctr-e50-1451931954322-0016-01-000002.dev.test.");
Record question = Record.newRecord(name, Type.A, DClass.IN);
Message query = Message.newQuery(question);
@@ -431,7 +431,7 @@ public class TestRegistryDNS extends Assert {
// KeyFactory keyFactory = KeyFactory.getInstance("RSA");
// PublicKey pubKey = keyFactory.generatePublic(keySpec);
DNSKEYRecord dnskeyRecord =
- new DNSKEYRecord(Name.fromString("hwxstg.site."), DClass.IN, 0,
+ new DNSKEYRecord(Name.fromString("dev.test."), DClass.IN, 0,
DNSKEYRecord.Flags.ZONE_KEY,
DNSKEYRecord.Protocol.DNSSEC,
DNSSEC.Algorithm.RSASHA256,
@@ -486,11 +486,11 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Record[] recs = assertDNSQuery(
- "ctr-e50-1451931954322-0016-01-000002.hwx.test.", Type.AAAA, 1);
+ "ctr-e50-1451931954322-0016-01-000002.dev.test.", Type.AAAA, 1);
assertEquals("wrong result", "172.17.0.19",
((AAAARecord) recs[0]).getAddress().getHostAddress());
- recs = assertDNSQuery("comp-name.test1.root.hwx.test.", Type.AAAA, 1);
+ recs = assertDNSQuery("comp-name.test1.root.dev.test.", Type.AAAA, 1);
assertTrue("not an ARecord", recs[0] instanceof AAAARecord);
}
@@ -504,7 +504,7 @@ public class TestRegistryDNS extends Assert {
record);
// start assessing whether correct records are available
- Name name = Name.fromString("missing.hwx.test.");
+ Name name = Name.fromString("missing.dev.test.");
Record question = Record.newRecord(name, Type.A, DClass.IN);
Message query = Message.newQuery(question);
@@ -533,7 +533,7 @@ public class TestRegistryDNS extends Assert {
public void testReadMasterFile() throws Exception {
setRegistryDNS(new RegistryDNS("TestRegistry"));
Configuration conf = new Configuration();
- conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test");
conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0");
conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
conf.set(RegistryConstants.KEY_DNS_ZONES_DIR,
@@ -561,17 +561,17 @@ public class TestRegistryDNS extends Assert {
// start assessing whether correct records are available
Record[] recs =
- assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.hwx.test.");
+ assertDNSQuery("ctr-e50-1451931954322-0016-01-000002.dev.test.");
assertEquals("wrong result", "172.17.0.19",
((ARecord) recs[0]).getAddress().getHostAddress());
- recs = assertDNSQuery("comp-name.test1.root.hwx.test.", 1);
+ recs = assertDNSQuery("comp-name.test1.root.dev.test.", 1);
assertTrue("not an ARecord", recs[0] instanceof ARecord);
// lookup dyanmic reverse records
recs = assertDNSQuery("19.0.17.172.in-addr.arpa.", Type.PTR, 1);
assertEquals("wrong result",
- "comp-name.test1.root.hwx.test.",
+ "comp-name.test1.root.dev.test.",
((PTRRecord) recs[0]).getTarget().toString());
// now lookup static reverse records
@@ -583,7 +583,7 @@ public class TestRegistryDNS extends Assert {
byte[] responseBytes = getRegistryDNS().generateReply(query, null);
Message response = new Message(responseBytes);
recs = response.getSectionArray(Section.ANSWER);
- assertEquals("wrong result", "cn005.hwx.test.",
+ assertEquals("wrong result", "cn005.dev.test.",
((PTRRecord) recs[0]).getTarget().toString());
}
@@ -636,7 +636,7 @@ public class TestRegistryDNS extends Assert {
public void testExternalCNAMERecord() throws Exception {
setRegistryDNS(new RegistryDNS("TestRegistry"));
Configuration conf = new Configuration();
- conf.set(RegistryConstants.KEY_DNS_DOMAIN, "hwx.test");
+ conf.set(RegistryConstants.KEY_DNS_DOMAIN, "dev.test");
conf.set(RegistryConstants.KEY_DNS_ZONE_SUBNET, "172.17.0");
conf.setTimeDuration(RegistryConstants.KEY_DNS_TTL, 30L, TimeUnit.SECONDS);
conf.set(RegistryConstants.KEY_DNS_ZONES_DIR,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d185d62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
index 0165f0d..08071e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/resources/0.17.172.in-addr.arpa.zone
@@ -19,18 +19,18 @@
$ORIGIN .
$TTL 1800 ; 30 minutes
0.17.172.in-addr.arpa IN SOA ns.hwhq.hortonworks.com. it.hortonworks.com. (
- 2015081000 ; serial
- 10800 ; refresh (3 hours)
- 900 ; retry (15 minutes)
- 1814400 ; expire (3 weeks)
- 10800 ; minimum (3 hours)
+ 2015081000 ; serial
+ 10800 ; refresh (3 hours)
+ 900 ; retry (15 minutes)
+ 1814400 ; expire (3 weeks)
+ 10800 ; minimum (3 hours)
)
- NS ns.hwhq.hortonworks.com.
- NS ns2.hwhq.hortonworks.com.
+ NS ns.hwhq.hortonworks.com.
+ NS ns2.hwhq.hortonworks.com.
$ORIGIN 0.17.172.in-addr.arpa.
-5 PTR cn005.hwx.test.
-6 PTR cn006.hwx.test.
-7 PTR cn007.hwx.test.
-8 PTR cn008.hwx.test.
-9 PTR cn009.hwx.test.
+5 PTR cn005.dev.test.
+6 PTR cn006.dev.test.
+7 PTR cn007.dev.test.
+8 PTR cn008.dev.test.
+9 PTR cn009.dev.test.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d185d62/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
index 53c5fea..7ec2ecb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Configurations.md
@@ -142,12 +142,12 @@ User can use these constants in their configurations to be dynamically substitut
```
Here, `COMPONENT_INSTANCE_NAME` and `SERVICE_NAME` are the constants to be substituted by the system.
-Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, user name is `devuser` and domain name is `hwxdev.site`.
+Suppose the `COMPONENT_INSTANCE_NAME` is `regionserver-0` and `SERVICE_NAME` is defined by user as `hbase`, user name is `devuser` and domain name is `dev.test`.
Then, the config will be substituted by the service AM and written in the config file `/etc/hadoop/hbase-site.xml` inside the container as below:
```
<property>
<name>hbase.regionserver.hostname</name>
- <value>regionserver-0.hbase.devuser.hwxdev.site</value>
+ <value>regionserver-0.hbase.devuser.dev.test</value>
</property>
```
where `regionserver-0` is the actual component instance name assigned by the system for this container.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-8018. Added support for
initiating yarn service upgrade. Contributed by Chandni Singh
Posted by na...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
index 091e624..df6d642 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/TestYarnNativeServices.java
@@ -33,7 +33,7 @@ import org.apache.hadoop.yarn.service.api.records.Component;
import org.apache.hadoop.yarn.service.api.records.Container;
import org.apache.hadoop.yarn.service.api.records.ContainerState;
import org.apache.hadoop.yarn.service.client.ServiceClient;
-import org.apache.hadoop.yarn.service.exceptions.SliderException;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
import org.apache.hadoop.yarn.service.utils.SliderFileSystem;
import org.hamcrest.CoreMatchers;
import org.junit.After;
@@ -86,7 +86,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
@Test (timeout = 200000)
public void testCreateFlexStopDestroyService() throws Exception {
setupInternal(NUM_NMS);
- ServiceClient client = createClient();
+ ServiceClient client = createClient(getConf());
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
SliderFileSystem fileSystem = new SliderFileSystem(getConf());
@@ -141,7 +141,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
@Test (timeout = 200000)
public void testComponentStartOrder() throws Exception {
setupInternal(NUM_NMS);
- ServiceClient client = createClient();
+ ServiceClient client = createClient(getConf());
Service exampleApp = new Service();
exampleApp.setName("teststartorder");
exampleApp.setVersion("v1");
@@ -169,7 +169,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
String userB = "userb";
setupInternal(NUM_NMS);
- ServiceClient client = createClient();
+ ServiceClient client = createClient(getConf());
String origBasePath = getConf().get(YARN_SERVICE_BASE_PATH);
Service userAApp = new Service();
@@ -221,7 +221,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
System.setProperty("user.name", user);
setupInternal(NUM_NMS);
- ServiceClient client = createClient();
+ ServiceClient client = createClient(getConf());
Service appA = new Service();
appA.setName(sameAppName);
@@ -290,7 +290,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
setConf(conf);
setupInternal(NUM_NMS);
- ServiceClient client = createClient();
+ ServiceClient client = createClient(getConf());
Service exampleApp = createExampleApplication();
client.actionCreate(exampleApp);
Multimap<String, String> containersBeforeFailure =
@@ -333,6 +333,28 @@ public class TestYarnNativeServices extends ServiceTestUtils {
client.actionDestroy(exampleApp.getName());
}
+ @Test(timeout = 200000)
+ public void testUpgradeService() throws Exception {
+ setupInternal(NUM_NMS);
+ ServiceClient client = createClient(getConf());
+
+ Service service = createExampleApplication();
+ client.actionCreate(service);
+ waitForServiceToBeStarted(client, service);
+
+ //upgrade the service
+ service.setVersion("v2");
+ client.actionUpgrade(service);
+
+ //wait for service to be in upgrade state
+ waitForServiceToBeInState(client, service, ServiceState.UPGRADING);
+ SliderFileSystem fs = new SliderFileSystem(getConf());
+ Service fromFs = ServiceApiUtil.loadServiceUpgrade(fs,
+ service.getName(), service.getVersion());
+ Assert.assertEquals(service.getName(), fromFs.getName());
+ Assert.assertEquals(service.getVersion(), fromFs.getVersion());
+ }
+
// Check containers launched are in dependency order
// Get all containers into a list and sort based on container launch time e.g.
// compa-c1, compa-c2, compb-c1, compb-c2;
@@ -470,16 +492,7 @@ public class TestYarnNativeServices extends ServiceTestUtils {
*/
private void waitForServiceToBeStable(ServiceClient client,
Service exampleApp) throws TimeoutException, InterruptedException {
- GenericTestUtils.waitFor(() -> {
- try {
- Service retrievedApp = client.getStatus(exampleApp.getName());
- System.out.println(retrievedApp);
- return retrievedApp.getState() == ServiceState.STABLE;
- } catch (Exception e) {
- e.printStackTrace();
- return false;
- }
- }, 2000, 200000);
+ waitForServiceToBeInState(client, exampleApp, ServiceState.STABLE);
}
/**
@@ -492,11 +505,25 @@ public class TestYarnNativeServices extends ServiceTestUtils {
*/
private void waitForServiceToBeStarted(ServiceClient client,
Service exampleApp) throws TimeoutException, InterruptedException {
+ waitForServiceToBeInState(client, exampleApp, ServiceState.STARTED);
+ }
+
+ /**
+ * Wait until service is started. It does not have to reach a stable state.
+ *
+ * @param client
+ * @param exampleApp
+ * @throws TimeoutException
+ * @throws InterruptedException
+ */
+ private void waitForServiceToBeInState(ServiceClient client,
+ Service exampleApp, ServiceState desiredState) throws TimeoutException,
+ InterruptedException {
GenericTestUtils.waitFor(() -> {
try {
Service retrievedApp = client.getStatus(exampleApp.getName());
System.out.println(retrievedApp);
- return retrievedApp.getState() == ServiceState.STARTED;
+ return retrievedApp.getState() == desiredState;
} catch (Exception e) {
e.printStackTrace();
return false;
@@ -504,21 +531,6 @@ public class TestYarnNativeServices extends ServiceTestUtils {
}, 2000, 200000);
}
- private ServiceClient createClient() throws Exception {
- ServiceClient client = new ServiceClient() {
- @Override protected Path addJarResource(String appName,
- Map<String, LocalResource> localResources)
- throws IOException, SliderException {
- // do nothing, the Unit test will use local jars
- return null;
- }
- };
- client.init(getConf());
- client.start();
- return client;
- }
-
-
private int countTotalContainers(Service service) {
int totalContainers = 0;
for (Component component : service.getComponents()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java
new file mode 100644
index 0000000..cc5b6ec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/client/TestServiceClient.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.client;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.service.ClientAMProtocol;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
+import org.apache.hadoop.yarn.service.api.records.Service;
+import org.apache.hadoop.yarn.service.utils.ServiceApiUtil;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.mockito.Matchers;
+
+import java.io.IOException;
+import java.util.ArrayList;
+
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Tests for {@link ServiceClient}.
+ */
+public class TestServiceClient {
+
+ @Rule
+ public ServiceTestUtils.ServiceFSWatcher rule =
+ new ServiceTestUtils.ServiceFSWatcher();
+
+ @Test
+ public void testActionUpgrade() throws Exception {
+ ApplicationId applicationId = ApplicationId.newInstance(
+ System.currentTimeMillis(), 1);
+ ServiceClient client = createServiceClient(applicationId);
+
+ Service service = ServiceTestUtils.createExampleApplication();
+ service.setVersion("v1");
+ client.actionCreate(service);
+
+ //upgrade the service
+ service.setVersion("v2");
+ client.actionUpgrade(service);
+
+ //wait for service to be in upgrade state
+ Service fromFs = ServiceApiUtil.loadServiceUpgrade(rule.getFs(),
+ service.getName(), service.getVersion());
+ Assert.assertEquals(service.getName(), fromFs.getName());
+ Assert.assertEquals(service.getVersion(), fromFs.getVersion());
+ }
+
+
+ private ServiceClient createServiceClient(ApplicationId applicationId)
+ throws Exception {
+ ClientAMProtocol amProxy = mock(ClientAMProtocol.class);
+ YarnClient yarnClient = createMockYarnClient();
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(
+ applicationId, 1);
+ ApplicationAttemptReport attemptReport =
+ ApplicationAttemptReport.newInstance(attemptId, "localhost", 0,
+ null, null, null,
+ YarnApplicationAttemptState.RUNNING, null);
+
+ ApplicationReport appReport = mock(ApplicationReport.class);
+ when(appReport.getHost()).thenReturn("localhost");
+
+ when(yarnClient.getApplicationAttemptReport(Matchers.any()))
+ .thenReturn(attemptReport);
+ when(yarnClient.getApplicationReport(applicationId)).thenReturn(appReport);
+
+ ServiceClient client = new ServiceClient() {
+ @Override
+ protected void serviceInit(Configuration configuration) throws Exception {
+ }
+
+ @Override
+ protected ClientAMProtocol createAMProxy(String serviceName,
+ ApplicationReport appReport) throws IOException, YarnException {
+ return amProxy;
+ }
+
+ @Override
+ ApplicationId submitApp(Service app) throws IOException, YarnException {
+ return applicationId;
+ }
+ };
+
+ client.setFileSystem(rule.getFs());
+ client.setYarnClient(yarnClient);
+
+ client.init(rule.getConf());
+ client.start();
+ return client;
+ }
+
+ private YarnClient createMockYarnClient() throws IOException, YarnException {
+ YarnClient yarnClient = mock(YarnClient.class);
+ when(yarnClient.getApplications(Matchers.any(GetApplicationsRequest.class)))
+ .thenReturn(new ArrayList<>());
+ return yarnClient;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestCoreFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestCoreFileSystem.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestCoreFileSystem.java
new file mode 100644
index 0000000..ba4a658
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-services/hadoop-yarn-services-core/src/test/java/org/apache/hadoop/yarn/service/utils/TestCoreFileSystem.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.service.utils;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.ServiceTestUtils;
+import org.apache.hadoop.yarn.service.conf.YarnServiceConstants;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+
+/**
+ * Tests for {@link CoreFileSystem}.
+ */
+public class TestCoreFileSystem {
+
+ @Rule
+ public ServiceTestUtils.ServiceFSWatcher rule =
+ new ServiceTestUtils.ServiceFSWatcher();
+
+ @Test
+ public void testClusterUpgradeDirPath() {
+ String serviceName = "testClusterUpgrade";
+ String version = "v1";
+ Path expectedPath = new Path(rule.getFs().buildClusterDirPath(serviceName),
+ YarnServiceConstants.UPGRADE_DIR + "/" + version);
+ Assert.assertEquals("incorrect upgrade path", expectedPath,
+ rule.getFs().buildClusterUpgradeDirPath(serviceName, version));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27d60a16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
index a09663e..d5eb787 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AppAdminClient.java
@@ -229,4 +229,20 @@ public abstract class AppAdminClient extends CompositeService {
@Unstable
public abstract String getStatusString(String appIdOrName) throws
IOException, YarnException;
+
+ /**
+ * Upgrade a long running service.
+ *
+ * @param appName the name of the application
+ * @param fileName specification of application upgrade to save.
+ *
+ * @return exit code
+ * @throws IOException IOException
+ * @throws YarnException exception in client or server
+ */
+ @Public
+ @Unstable
+ public abstract int actionUpgrade(String appName, String fileName)
+ throws IOException, YarnException;
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HADOOP-15253. Should update
maxQueueSize when refresh call queue. Contributed by Tao Jie.
Posted by na...@apache.org.
HADOOP-15253. Should update maxQueueSize when refresh call queue. Contributed by Tao Jie.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/acfd764f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/acfd764f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/acfd764f
Branch: refs/heads/YARN-3409
Commit: acfd764fcc9990e507c0e7cea746652375aaa632
Parents: 6590747
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Fri Mar 30 18:55:35 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Mar 30 18:55:35 2018 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 3 +++
.../org/apache/hadoop/TestRefreshCallQueue.java | 17 +++++++++++++++--
2 files changed, 18 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acfd764f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index d0694fb..c5da3b1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -657,6 +657,9 @@ public abstract class Server {
public synchronized void refreshCallQueue(Configuration conf) {
// Create the next queue
String prefix = getQueueClassPrefix();
+ this.maxQueueSize = handlerCount * conf.getInt(
+ CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,
+ CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT);
callQueue.swapQueue(getSchedulerClass(prefix, conf),
getQueueClass(prefix, conf), maxQueueSize, prefix, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/acfd764f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
index d5eb9cf..faac1c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/TestRefreshCallQueue.java
@@ -30,8 +30,10 @@ import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer;
import org.apache.hadoop.hdfs.tools.DFSAdmin;
@@ -141,8 +143,16 @@ public class TestRefreshCallQueue {
// throw an error when we double-initialize JvmMetrics
DefaultMetricsSystem.setMiniClusterMode(false);
-
+ int serviceHandlerCount = config.getInt(
+ DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_KEY,
+ DFSConfigKeys.DFS_NAMENODE_SERVICE_HANDLER_COUNT_DEFAULT);
NameNodeRpcServer rpcServer = (NameNodeRpcServer) cluster.getNameNodeRpc();
+ // check callqueue size
+ assertEquals(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_DEFAULT
+ * serviceHandlerCount, rpcServer.getClientRpcServer().getMaxQueueSize());
+ // Replace queue and update queue size
+ config.setInt(CommonConfigurationKeys.IPC_SERVER_HANDLER_QUEUE_SIZE_KEY,
+ 150);
try {
rpcServer.getClientRpcServer().refreshCallQueue(config);
} catch (Exception e) {
@@ -158,6 +168,9 @@ public class TestRefreshCallQueue {
} finally {
DefaultMetricsSystem.setMiniClusterMode(oldValue);
}
- }
+ // check callQueueSize has changed
+ assertEquals(150 * serviceHandlerCount, rpcServer.getClientRpcServer()
+ .getMaxQueueSize());
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-8092. Expose Node Attributes
info via RM nodes REST API. Contributed by Weiwei Yang.
Posted by na...@apache.org.
YARN-8092. Expose Node Attributes info via RM nodes REST API. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4724556b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4724556b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4724556b
Branch: refs/heads/YARN-3409
Commit: 4724556be5fb16e2a6f2582361eb9b9b4fc76ecb
Parents: 33b67d0
Author: Sunil G <su...@apache.org>
Authored: Sun Apr 1 18:07:07 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 14:11:12 2018 +0800
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 13 ++++
.../yarn/sls/scheduler/RMNodeWrapper.java | 12 ++++
.../resourcemanager/ResourceTrackerService.java | 4 ++
.../server/resourcemanager/rmnode/RMNode.java | 13 ++++
.../resourcemanager/rmnode/RMNodeImpl.java | 15 +++++
.../webapp/dao/NodeAttributeInfo.java | 65 ++++++++++++++++++++
.../webapp/dao/NodeAttributesInfo.java | 49 +++++++++++++++
.../resourcemanager/webapp/dao/NodeInfo.java | 15 +++++
.../yarn/server/resourcemanager/MockNodes.java | 11 ++++
9 files changed, 197 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 0c99139..29d54a1 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -218,6 +219,18 @@ public class NodeInfo {
return null;
}
+
+ @Override
+ public void setNodeAttributes(String prefix,
+ Set<NodeAttribute> nodeAttributes) {
+
+ }
+
+ @Override
+ public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
+ return null;
+ }
+
@Override
public Resource getPhysicalResource() {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 78645e9..a603036 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -205,6 +206,17 @@ public class RMNodeWrapper implements RMNode {
public Map<String, Long> getAllocationTagsWithCount() {
return node.getAllocationTagsWithCount();
}
+
+ @Override
+ public void setNodeAttributes(String prefix,
+ Set<NodeAttribute> nodeAttributes) {
+ node.setNodeAttributes(prefix, nodeAttributes);
+ }
+
+ @Override
+ public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
+ return node.getAllNodeAttributes();
+ }
@Override
public Resource getPhysicalResource() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 8a1a9a7..4f4400f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -673,6 +673,10 @@ public class ResourceTrackerService extends AbstractService implements
this.rmContext.getNodeAttributesManager()
.replaceNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
ImmutableMap.of(nodeId.getHost(), nodeAttributes));
+
+ // Update node attributes to RMNode
+ rmNode.setNodeAttributes(NodeAttribute.PREFIX_DISTRIBUTED,
+ nodeAttributes);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 872f2a6..8d3b871 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
@@ -189,4 +190,16 @@ public interface RMNode {
* @return a map of each allocation tag and its count.
*/
Map<String, Long> getAllocationTagsWithCount();
+
+ /**
+ * Sets node attributes per prefix.
+ * @param prefix node attribute prefix
+ * @param nodeAttributes node attributes
+ */
+ void setNodeAttributes(String prefix, Set<NodeAttribute> nodeAttributes);
+
+ /**
+ * @return all node attributes grouped by their prefix as a map.
+ */
+ Map<String, Set<NodeAttribute>> getAllNodeAttributes();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b942afa..f5bcd6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.ContainerUpdateType;
import org.apache.hadoop.yarn.api.records.ExecutionType;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -185,6 +186,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private NodeHeartbeatResponse latestNodeHeartBeatResponse = recordFactory
.newRecordInstance(NodeHeartbeatResponse.class);
+ // Node attributes, store by prefix
+ private Map<String, Set<NodeAttribute>> nodeAttributes = new HashMap<>();
+
private static final StateMachineFactory<RMNodeImpl,
NodeState,
RMNodeEventType,
@@ -1541,4 +1545,15 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
return context.getAllocationTagsManager()
.getAllocationTagsWithCount(getNodeID());
}
+
+ @Override
+ public void setNodeAttributes(String prefix,
+ Set<NodeAttribute> nodeAttributeSet) {
+ this.nodeAttributes.put(prefix, nodeAttributeSet);
+ }
+
+ @Override
+ public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
+ return this.nodeAttributes;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
new file mode 100644
index 0000000..bbc2ec3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+/**
+ * DAO for node an attribute record.
+ */
+@XmlRootElement(name = "nodeAttributeInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeAttributeInfo {
+
+ private String prefix;
+ private String name;
+ private String type;
+ private String value;
+
+ public NodeAttributeInfo() {
+ // JAXB needs this
+ }
+
+ public NodeAttributeInfo(NodeAttribute nodeAttribute) {
+ this.prefix = nodeAttribute.getAttributePrefix();
+ this.name = nodeAttribute.getAttributeName();
+ this.type = nodeAttribute.getAttributeType().toString();
+ this.value = nodeAttribute.getAttributeValue();
+ }
+
+ public String getPrefix() {
+ return prefix;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public String getType() {
+ return type;
+ }
+
+ public String getValue() {
+ return value;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributesInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributesInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributesInfo.java
new file mode 100644
index 0000000..1f4c25f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributesInfo.java
@@ -0,0 +1,49 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+import java.util.ArrayList;
+
+/**
+ * DAO for a list of node attributes info.
+ */
+@XmlRootElement(name = "nodeAttributesInfo")
+@XmlAccessorType(XmlAccessType.FIELD)
+public class NodeAttributesInfo {
+
+ @XmlElement(name = "nodeAttributeInfo")
+ private ArrayList<NodeAttributeInfo> nodeAttributesInfo =
+ new ArrayList<>();
+
+ public NodeAttributesInfo() {
+ // JAXB needs this
+ }
+
+ public void addNodeAttributeInfo(NodeAttributeInfo attributeInfo) {
+ this.nodeAttributesInfo.add(attributeInfo);
+ }
+
+ public ArrayList<NodeAttributeInfo> getNodeAttributesInfo() {
+ return nodeAttributesInfo;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 46a6e60..ecf66b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
@@ -62,6 +63,7 @@ public class NodeInfo {
protected ResourceUtilizationInfo resourceUtilization;
protected ResourceInfo usedResource;
protected ResourceInfo availableResource;
+ protected NodeAttributesInfo nodeAttributesInfo;
public NodeInfo() {
} // JAXB needs this
@@ -113,6 +115,19 @@ public class NodeInfo {
Collections.sort(nodeLabels);
}
+ // add attributes
+ Map<String, Set<NodeAttribute>> nodeAttributes =
+ ni.getAllNodeAttributes();
+ nodeAttributesInfo = new NodeAttributesInfo();
+ if (nodeAttributes != null) {
+ for (Set<NodeAttribute> attrs : nodeAttributes.values()) {
+ for (NodeAttribute attribute : attrs) {
+ NodeAttributeInfo info = new NodeAttributeInfo(attribute);
+ this.nodeAttributesInfo.addNodeAttributeInfo(info);
+ }
+ }
+ }
+
// add allocation tags
allocationTags = new AllocationTagsInfo();
Map<String, Long> allocationTagsInfo = ni.getAllocationTagsWithCount();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4724556b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 84105d9..a92a574 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.net.Node;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -284,6 +285,16 @@ public class MockNodes {
public Map<String, Long> getAllocationTagsWithCount() {
return null;
}
+
+ public void setNodeAttributes(String prefix,
+ Set<NodeAttribute> nodeAttributes) {
+
+ }
+
+ @Override
+ public Map<String, Set<NodeAttribute>> getAllNodeAttributes() {
+ return null;
+ }
@Override
public Resource getPhysicalResource() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDFS-13338. Update BUILDING.txt
for building native libraries. Contributed by Takanobu Asanuma.
Posted by na...@apache.org.
HDFS-13338. Update BUILDING.txt for building native libraries. Contributed by Takanobu Asanuma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/285bbaa4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/285bbaa4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/285bbaa4
Branch: refs/heads/YARN-3409
Commit: 285bbaa4329f7586b9c404ce2e557d36041099c0
Parents: 4cef011
Author: James Clampffer <ja...@hp.com>
Authored: Tue Mar 27 14:21:19 2018 -0400
Committer: James Clampffer <ja...@hp.com>
Committed: Tue Mar 27 14:21:19 2018 -0400
----------------------------------------------------------------------
BUILDING.txt | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/285bbaa4/BUILDING.txt
----------------------------------------------------------------------
diff --git a/BUILDING.txt b/BUILDING.txt
index f1df194..dbf2cb8 100644
--- a/BUILDING.txt
+++ b/BUILDING.txt
@@ -9,6 +9,7 @@ Requirements:
* ProtocolBuffer 2.5.0
* CMake 3.1 or newer (if compiling native code)
* Zlib devel (if compiling native code)
+* Cyrus SASL devel (if compiling native code)
* openssl devel (if compiling native hadoop-pipes and to get the best HDFS encryption performance)
* Linux FUSE (Filesystem in Userspace) version 2.6 or above (if compiling fuse_dfs)
* Jansson C XML parsing library ( if compiling libwebhdfs )
@@ -67,7 +68,7 @@ Installing required packages for clean install of Ubuntu 14.04 LTS Desktop:
* Maven
$ sudo apt-get -y install maven
* Native libraries
- $ sudo apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev
+ $ sudo apt-get -y install build-essential autoconf automake libtool cmake zlib1g-dev pkg-config libssl-dev libsasl2-dev
* ProtocolBuffer 2.5.0 (required)
$ sudo apt-get -y install protobuf-compiler
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-7623. Fix the
CapacityScheduler Queue configuration documentation. Contributed by Jonathan
Hung.
Posted by na...@apache.org.
YARN-7623. Fix the CapacityScheduler Queue configuration documentation. Contributed by Jonathan Hung.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b1c2b5f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b1c2b5f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b1c2b5f
Branch: refs/heads/YARN-3409
Commit: 0b1c2b5fe1b5c225d208936ecb1d3e307a535ee6
Parents: 09999d7
Author: Zhe Zhang <ze...@linkedin.com>
Authored: Wed Mar 28 11:41:13 2018 -0700
Committer: Zhe Zhang <ze...@linkedin.com>
Committed: Wed Mar 28 11:41:20 2018 -0700
----------------------------------------------------------------------
.../src/site/markdown/CapacityScheduler.md | 16 ++++++++--------
1 file changed, 8 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b1c2b5f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
index 3fadff8..e1bb4b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/CapacityScheduler.md
@@ -426,14 +426,14 @@ Changing queue/scheduler properties and adding/removing queues can be done in tw
**Note:** This feature is in alpha phase and is subject to change.
- | Property | Description |
- |:---- |:---- |
- | `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). |
- | `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. |
- | `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. |
- | `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. |
- | `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. |
- | `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. |
+| Property | Description |
+|:---- |:---- |
+| `yarn.scheduler.configuration.store.class` | The type of backing store to use, as described [above](CapacityScheduler.html#Changing_Queue_Configuration). |
+| `yarn.scheduler.configuration.mutation.acl-policy.class` | An ACL policy can be configured to restrict which users can modify which queues. Default value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.DefaultConfigurationMutationACLPolicy*, which only allows YARN admins to make any configuration modifications. Another value is *org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.conf.QueueAdminConfigurationMutationACLPolicy*, which only allows queue modifications if the caller is an admin of the queue. |
+| `yarn.scheduler.configuration.store.max-logs` | Configuration changes are audit logged in the backing store, if using leveldb or zookeeper. This configuration controls the maximum number of audit logs to store, dropping the oldest logs when exceeded. Default is 1000. |
+| `yarn.scheduler.configuration.leveldb-store.path` | The storage path of the configuration store when using leveldb. Default value is *${hadoop.tmp.dir}/yarn/system/confstore*. |
+| `yarn.scheduler.configuration.leveldb-store.compaction-interval-secs` | The interval for compacting the configuration store in seconds, when using leveldb. Default value is 86400, or one day. |
+| `yarn.scheduler.configuration.zk-store.parent-path` | The zookeeper root node path for configuration store related information, when using zookeeper. Default value is */confstore*. |
**Note:** When enabling scheduler configuration mutations via `yarn.scheduler.configuration.store.class`, *yarn rmadmin -refreshQueues* will be disabled, i.e. it will no longer be possible to update configuration via file.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HADOOP-15342. Updating ADLS
connector to use the current SDK version (2.2.7). Contributed by Atul
Sikaria.
Posted by na...@apache.org.
HADOOP-15342. Updating ADLS connector to use the current SDK version (2.2.7). Contributed by Atul Sikaria.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9d7a9031
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9d7a9031
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9d7a9031
Branch: refs/heads/YARN-3409
Commit: 9d7a9031a5978efc8d97566e35ebaace20db2353
Parents: 7a59d60
Author: Sean Mackrory <ma...@apache.org>
Authored: Tue Mar 27 15:51:44 2018 -0600
Committer: Sean Mackrory <ma...@apache.org>
Committed: Thu Mar 29 10:13:40 2018 -0600
----------------------------------------------------------------------
hadoop-tools/hadoop-azure-datalake/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9d7a9031/hadoop-tools/hadoop-azure-datalake/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/pom.xml b/hadoop-tools/hadoop-azure-datalake/pom.xml
index 8020381..57515b0 100644
--- a/hadoop-tools/hadoop-azure-datalake/pom.xml
+++ b/hadoop-tools/hadoop-azure-datalake/pom.xml
@@ -33,7 +33,7 @@
<minimalJsonVersion>0.9.1</minimalJsonVersion>
<file.encoding>UTF-8</file.encoding>
<downloadSources>true</downloadSources>
- <azure.data.lake.store.sdk.version>2.2.5</azure.data.lake.store.sdk.version>
+ <azure.data.lake.store.sdk.version>2.2.7</azure.data.lake.store.sdk.version>
</properties>
<build>
<plugins>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-6858. Attribute Manager to
store and provide node attributes in RM. Contributed by Naganarasimha G R.
Posted by na...@apache.org.
YARN-6858. Attribute Manager to store and provide node attributes in RM. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/06fbea4e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/06fbea4e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/06fbea4e
Branch: refs/heads/YARN-3409
Commit: 06fbea4e58ffc7d6655effbe486c99b72b577ec0
Parents: 980cc55
Author: Sunil G <su...@apache.org>
Authored: Fri Feb 23 08:01:58 2018 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Apr 2 11:30:55 2018 +0800
----------------------------------------------------------------------
.../records/impl/pb/NodeAttributePBImpl.java | 15 +-
.../hadoop/yarn/nodelabels/AbstractLabel.java | 71 +++
.../AttributeExpressionOperation.java | 26 +
.../hadoop/yarn/nodelabels/AttributeValue.java | 53 ++
.../nodelabels/CommonNodeLabelsManager.java | 24 +-
.../yarn/nodelabels/NodeAttributesManager.java | 99 ++++
.../hadoop/yarn/nodelabels/NodeLabelUtil.java | 97 ++++
.../hadoop/yarn/nodelabels/RMNodeAttribute.java | 104 ++++
.../hadoop/yarn/nodelabels/RMNodeLabel.java | 109 ++--
.../yarn/nodelabels/StringAttributeValue.java | 61 +++
.../nodemanager/NodeStatusUpdaterImpl.java | 16 +-
.../resourcemanager/RMActiveServiceContext.java | 14 +
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 11 +
.../server/resourcemanager/ResourceManager.java | 40 +-
.../nodelabels/NodeAttributesManagerImpl.java | 527 +++++++++++++++++++
.../nodelabels/NodeAttributesStoreEvent.java | 51 ++
.../NodeAttributesStoreEventType.java | 26 +
18 files changed, 1230 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
index 7810939..12a0ecc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeAttributePBImpl.java
@@ -120,7 +120,13 @@ public class NodeAttributePBImpl extends NodeAttribute {
@Override
public int hashCode() {
- return getProto().hashCode();
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((getAttributePrefix() == null) ? 0
+ : getAttributePrefix().hashCode());
+ result = prime * result
+ + ((getAttributeName() == null) ? 0 : getAttributeName().hashCode());
+ return result;
}
@Override
@@ -133,13 +139,10 @@ public class NodeAttributePBImpl extends NodeAttribute {
}
if (obj instanceof NodeAttribute) {
NodeAttribute other = (NodeAttribute) obj;
- if (!compare(getAttributeName(), other.getAttributeName())) {
- return false;
- }
- if (!compare(getAttributeValue(), other.getAttributeValue())) {
+ if (!compare(getAttributePrefix(), other.getAttributePrefix())) {
return false;
}
- if (!compare(getAttributeType(), other.getAttributeType())) {
+ if (!compare(getAttributeName(), other.getAttributeName())) {
return false;
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
new file mode 100644
index 0000000..6a44574
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AbstractLabel.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * Generic class capturing the information required commonly across Partitions
+ * and Attributes.
+ */
+public abstract class AbstractLabel {
+
+ private Resource resource;
+ private int numActiveNMs;
+ private String labelName;
+
+ public AbstractLabel() {
+ super();
+ }
+
+ public AbstractLabel(String labelName) {
+ this(labelName, Resource.newInstance(0, 0), 0);
+ }
+
+ public AbstractLabel(String labelName, Resource resource, int numActiveNMs) {
+ super();
+ this.resource = resource;
+ this.numActiveNMs = numActiveNMs;
+ this.labelName = labelName;
+ }
+
+ public void addNode(Resource nodeRes) {
+ Resources.addTo(resource, nodeRes);
+ numActiveNMs++;
+ }
+
+ public void removeNode(Resource nodeRes) {
+ Resources.subtractFrom(resource, nodeRes);
+ numActiveNMs--;
+ }
+
+ public Resource getResource() {
+ return Resource.newInstance(this.resource);
+ }
+
+ public int getNumActiveNMs() {
+ return numActiveNMs;
+ }
+
+ public String getLabelName() {
+ return labelName;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeExpressionOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeExpressionOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeExpressionOperation.java
new file mode 100644
index 0000000..8754314
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeExpressionOperation.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+/**
+ * Operations which are allowed in Node Attributes Expression.
+ */
+public enum AttributeExpressionOperation {
+ LT, GT, IN, NOTIN
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java
new file mode 100644
index 0000000..d1d75cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/AttributeValue.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.io.IOException;
+
+/**
+ * Interface to capture operations on AttributeValue.
+ */
+public interface AttributeValue {
+
+ /**
+ * @return original value which was set.
+ */
+ String getValue();
+
+ /**
+ * validate the value based on the type and initialize for further compare
+ * operations.
+ *
+ * @param value
+ * @throws IOException
+ */
+ void validateAndInitializeValue(String value) throws IOException;
+
+ /**
+ * compare the value against the other based on the
+ * AttributeExpressionOperation.
+ *
+ * @param other
+ * @param op
+ * @return true if value <code>other</code> matches the current value for the
+ * operation <code>op</code>.
+ */
+ boolean compareForOperation(AttributeValue other,
+ AttributeExpressionOperation op);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 66e945f..24ee028 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -35,7 +35,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-import java.util.regex.Pattern;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
@@ -65,15 +64,12 @@ import com.google.common.collect.ImmutableSet;
@Private
public class CommonNodeLabelsManager extends AbstractService {
protected static final Log LOG = LogFactory.getLog(CommonNodeLabelsManager.class);
- private static final int MAX_LABEL_LENGTH = 255;
public static final Set<String> EMPTY_STRING_SET = Collections
.unmodifiableSet(new HashSet<String>(0));
public static final Set<NodeLabel> EMPTY_NODELABEL_SET = Collections
.unmodifiableSet(new HashSet<NodeLabel>(0));
public static final String ANY = "*";
public static final Set<String> ACCESS_ANY_LABEL_SET = ImmutableSet.of(ANY);
- private static final Pattern LABEL_PATTERN = Pattern
- .compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
public static final int WILDCARD_PORT = 0;
// Flag to identify startup for removelabel
private boolean initNodeLabelStoreInProgress = false;
@@ -112,7 +108,7 @@ public class CommonNodeLabelsManager extends AbstractService {
/**
* A <code>Host</code> can have multiple <code>Node</code>s
*/
- protected static class Host {
+ public static class Host {
public Set<String> labels;
public Map<NodeId, Node> nms;
@@ -317,7 +313,7 @@ public class CommonNodeLabelsManager extends AbstractService {
// do a check before actual adding them, will throw exception if any of them
// doesn't meet label name requirement
for (NodeLabel label : labels) {
- checkAndThrowLabelName(label.getName());
+ NodeLabelUtil.checkAndThrowLabelName(label.getName());
}
for (NodeLabel label : labels) {
@@ -969,22 +965,6 @@ public class CommonNodeLabelsManager extends AbstractService {
}
}
- public static void checkAndThrowLabelName(String label) throws IOException {
- if (label == null || label.isEmpty() || label.length() > MAX_LABEL_LENGTH) {
- throw new IOException("label added is empty or exceeds "
- + MAX_LABEL_LENGTH + " character(s)");
- }
- label = label.trim();
-
- boolean match = LABEL_PATTERN.matcher(label).matches();
-
- if (!match) {
- throw new IOException("label name should only contains "
- + "{0-9, a-z, A-Z, -, _} and should not started with {-,_}"
- + ", now it is=" + label);
- }
- }
-
private void checkExclusivityMatch(Collection<NodeLabel> labels)
throws IOException {
ArrayList<NodeLabel> mismatchlabels = new ArrayList<NodeLabel>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
new file mode 100644
index 0000000..63f3dcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeAttributesManager.java
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+
+/**
+ * This class captures all interactions for Attributes with RM.
+ */
+public abstract class NodeAttributesManager extends AbstractService {
+ public NodeAttributesManager(String name) {
+ super(name);
+ }
+
+ /**
+ * To completely replace the mappings for a given node with the new Set of
+ * Attributes. If the mapping contains an attribute whose type does not match
+ * a previously existing Attribute under the same prefix (name space) then
+ * exception is thrown. Key would be name of the node and value would be set
+ * of Attributes to be mapped.
+ *
+ * @param nodeAttributeMapping
+ * @throws IOException
+ */
+ public abstract void replaceNodeAttributes(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
+
+ /**
+ * It adds or updates the attribute mapping for a given node with out
+ * impacting other existing attribute mapping. Key would be name of the node
+ * and value would be set of Attributes to be mapped.
+ *
+ * @param nodeAttributeMapping
+ * @throws IOException
+ */
+ public abstract void addNodeAttributes(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
+
+ /**
+ * It removes the specified attribute mapping for a given node with out
+ * impacting other existing attribute mapping. Key would be name of the node
+ * and value would be set of Attributes to be removed.
+ *
+ * @param nodeAttributeMapping
+ * @throws IOException
+ */
+ public abstract void removeNodeAttributes(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException;
+
+ /**
+ * @param prefix set of prefix string's for which the attributes needs to
+ * returned
+ * @return set of node Attributes objects for the specified set of prefixes,
+ * else return all
+ */
+ public abstract Set<NodeAttribute> getClusterNodeAttributes(
+ Set<String> prefix);
+
+ /**
+ * Given a attribute set, return what all Nodes have attribute mapped to it.
+ *
+ * @return a Map, of attribute to set of hostnames
+ */
+ //TODO need to handle as part of REST patch.
+/* public abstract Map<NodeAttribute, Set<String>> getAttributesToNodes(
+ Set<NodeAttribute> attributes);*/
+
+ /**
+ * NodeAttribute to AttributeValue Map.
+ *
+ * @return Map<NodeAttribute, AttributeValue> mapping of Attribute to Value.
+ */
+ public abstract Map<NodeAttribute, AttributeValue> getAttributesForNode(
+ String hostName);
+
+ // futuristic
+ // public set<NodeId> getNodesMatchingExpression(String nodeLabelExp);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
new file mode 100644
index 0000000..d918712
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelUtil.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+/**
+ * Utility class for all NodeLabel and NodeAttribute operations.
+ */
+public final class NodeLabelUtil {
+ private NodeLabelUtil() {
+ }
+
+ private static final int MAX_LABEL_LENGTH = 255;
+ private static final Pattern LABEL_OR_VALUE_PATTERN =
+ Pattern.compile("^[0-9a-zA-Z][0-9a-zA-Z-_]*");
+ private static final Pattern PREFIX_PATTERN =
+ Pattern.compile("^[0-9a-zA-Z][0-9a-zA-Z-_\\.]*");
+
+ public static void checkAndThrowLabelName(String label) throws IOException {
+ if (label == null || label.isEmpty() || label.length() > MAX_LABEL_LENGTH) {
+ throw new IOException("label added is empty or exceeds "
+ + MAX_LABEL_LENGTH + " character(s)");
+ }
+ label = label.trim();
+
+ boolean match = LABEL_OR_VALUE_PATTERN.matcher(label).matches();
+
+ if (!match) {
+ throw new IOException("label name should only contains "
+ + "{0-9, a-z, A-Z, -, _} and should not started with {-,_}"
+ + ", now it is= " + label);
+ }
+ }
+
+ public static void checkAndThrowAttributeValue(String value)
+ throws IOException {
+ if (value == null) {
+ return;
+ } else if (value.trim().length() > MAX_LABEL_LENGTH) {
+ throw new IOException("Attribute value added exceeds " + MAX_LABEL_LENGTH
+ + " character(s)");
+
+ }
+ value = value.trim();
+ if(value.isEmpty()) {
+ return;
+ }
+
+ boolean match = LABEL_OR_VALUE_PATTERN.matcher(value).matches();
+
+ if (!match) {
+ throw new IOException("attribute value should only contains "
+ + "{0-9, a-z, A-Z, -, _} and should not started with {-,_}"
+ + ", now it is= " + value);
+ }
+ }
+
+ public static void checkAndThrowAttributePrefix(String prefix)
+ throws IOException {
+ if (prefix == null) {
+ throw new IOException("Attribute prefix cannot be null.");
+ }
+ if (prefix.trim().length() > MAX_LABEL_LENGTH) {
+ throw new IOException("Attribute value added exceeds " + MAX_LABEL_LENGTH
+ + " character(s)");
+ }
+ prefix = prefix.trim();
+ if(prefix.isEmpty()) {
+ return;
+ }
+
+ boolean match = PREFIX_PATTERN.matcher(prefix).matches();
+
+ if (!match) {
+ throw new IOException("attribute value should only contains "
+ + "{0-9, a-z, A-Z, -, _,.} and should not started with {-,_}"
+ + ", now it is= " + prefix);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java
new file mode 100644
index 0000000..5a709c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeAttribute.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * Reference of NodeAttribute in RM.
+ */
+public class RMNodeAttribute extends AbstractLabel {
+
+ private NodeAttribute attribute;
+ // TODO need to revisit whether we need to make this concurrent implementation
+ private Set<String> nodes = new HashSet<>();
+
+ public RMNodeAttribute(NodeAttribute attribute) {
+ this(attribute.getAttributeName(), Resource.newInstance(0, 0), 0,
+ attribute);
+ }
+
+ public RMNodeAttribute(String labelName, Resource res, int activeNMs,
+ NodeAttribute attribute) {
+ super(labelName, res, activeNMs);
+ this.attribute = attribute;
+ }
+
+ public NodeAttribute getAttribute() {
+ return attribute;
+ }
+
+ public void setAttribute(NodeAttribute attribute) {
+ this.attribute = attribute;
+ }
+
+ public RMNodeAttribute(String attributeName) {
+ super(attributeName);
+ attribute = NodeAttribute.newInstance(attributeName,
+ NodeAttributeType.STRING, CommonNodeLabelsManager.NO_LABEL);
+ }
+
+ public NodeAttributeType getAttributeType() {
+ return attribute.getAttributeType();
+ }
+
+ public void addNode(String node) {
+ nodes.add(node);
+ }
+
+ public void removeNode(String node) {
+ nodes.remove(node);
+ }
+
+ public Set<String> getAssociatedNodeIds() {
+ return new HashSet<String>(nodes);
+ }
+
+ @Override
+ public int hashCode() {
+ return attribute.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ RMNodeAttribute other = (RMNodeAttribute) obj;
+ if (attribute == null) {
+ if (other.attribute != null) {
+ return false;
+ }
+ } else if (!attribute.equals(other.attribute)) {
+ return false;
+ }
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java
index feeeaf1..7f7db9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/RMNodeLabel.java
@@ -27,13 +27,13 @@ import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.util.resource.Resources;
-public class RMNodeLabel implements Comparable<RMNodeLabel> {
- private Resource resource;
- private int numActiveNMs;
- private String labelName;
- private Set<NodeId> nodeIds;
+/**
+ * Partition representation in RM.
+ */
+public class RMNodeLabel extends AbstractLabel implements Comparable<RMNodeLabel> {
private boolean exclusive;
private NodeLabel nodeLabel;
+ private Set<NodeId> nodeIds;
public RMNodeLabel(NodeLabel nodeLabel) {
this(nodeLabel.getName(), Resource.newInstance(0, 0), 0,
@@ -47,48 +47,12 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> {
protected RMNodeLabel(String labelName, Resource res, int activeNMs,
boolean exclusive) {
- this.labelName = labelName;
- this.resource = res;
- this.numActiveNMs = activeNMs;
- this.nodeIds = new HashSet<NodeId>();
+ super(labelName, res, activeNMs);
this.exclusive = exclusive;
this.nodeLabel = NodeLabel.newInstance(labelName, exclusive);
+ nodeIds = new HashSet<NodeId>();
}
- public void addNodeId(NodeId node) {
- nodeIds.add(node);
- }
-
- public void removeNodeId(NodeId node) {
- nodeIds.remove(node);
- }
-
- public Set<NodeId> getAssociatedNodeIds() {
- return new HashSet<NodeId>(nodeIds);
- }
-
- public void addNode(Resource nodeRes) {
- Resources.addTo(resource, nodeRes);
- numActiveNMs++;
- }
-
- public void removeNode(Resource nodeRes) {
- Resources.subtractFrom(resource, nodeRes);
- numActiveNMs--;
- }
-
- public Resource getResource() {
- return this.resource;
- }
-
- public int getNumActiveNMs() {
- return numActiveNMs;
- }
-
- public String getLabelName() {
- return labelName;
- }
-
public void setIsExclusive(boolean exclusive) {
this.exclusive = exclusive;
}
@@ -97,42 +61,57 @@ public class RMNodeLabel implements Comparable<RMNodeLabel> {
return this.exclusive;
}
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof RMNodeLabel) {
+ RMNodeLabel other = (RMNodeLabel) obj;
+ return Resources.equals(getResource(), other.getResource())
+ && StringUtils.equals(getLabelName(), other.getLabelName())
+ && (other.getNumActiveNMs() == getNumActiveNMs());
+ }
+ return false;
+ }
+
+
public RMNodeLabel getCopy() {
- return new RMNodeLabel(labelName, resource, numActiveNMs, exclusive);
+ return new RMNodeLabel(getLabelName(), getResource(), getNumActiveNMs(),
+ exclusive);
}
- public NodeLabel getNodeLabel() {
- return this.nodeLabel;
+ @Override
+ public int hashCode() {
+ final int prime = 502357;
+ return (int) ((((long) getLabelName().hashCode() << 8)
+ + (getResource().hashCode() << 4) + getNumActiveNMs()) % prime);
}
+
@Override
public int compareTo(RMNodeLabel o) {
// We should always put empty label entry first after sorting
- if (labelName.isEmpty() != o.getLabelName().isEmpty()) {
- if (labelName.isEmpty()) {
+ if (getLabelName().isEmpty() != o.getLabelName().isEmpty()) {
+ if (getLabelName().isEmpty()) {
return -1;
}
return 1;
}
- return labelName.compareTo(o.getLabelName());
+ return getLabelName().compareTo(o.getLabelName());
}
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof RMNodeLabel) {
- RMNodeLabel other = (RMNodeLabel) obj;
- return Resources.equals(resource, other.getResource())
- && StringUtils.equals(labelName, other.getLabelName())
- && (other.getNumActiveNMs() == numActiveNMs);
- }
- return false;
+
+ public NodeLabel getNodeLabel() {
+ return this.nodeLabel;
}
-
- @Override
- public int hashCode() {
- final int prime = 502357;
- return (int) ((((long) labelName.hashCode() << 8)
- + (resource.hashCode() << 4) + numActiveNMs) % prime);
+
+ public void addNodeId(NodeId node) {
+ nodeIds.add(node);
+ }
+
+ public void removeNodeId(NodeId node) {
+ nodeIds.remove(node);
+ }
+
+ public Set<NodeId> getAssociatedNodeIds() {
+ return new HashSet<NodeId>(nodeIds);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/StringAttributeValue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/StringAttributeValue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/StringAttributeValue.java
new file mode 100644
index 0000000..12343a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/StringAttributeValue.java
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.io.IOException;
+
+/**
+ * Attribute value for String NodeAttributeType.
+ */
+public class StringAttributeValue implements AttributeValue {
+ private String value = "";
+
+ @Override
+ public boolean compareForOperation(AttributeValue other,
+ AttributeExpressionOperation op) {
+ if (other instanceof StringAttributeValue) {
+ StringAttributeValue otherString = (StringAttributeValue) other;
+ switch (op) {
+ case IN:
+ return value.equals(otherString.value);
+ case NOTIN:
+ return !value.equals(otherString.value);
+ default:
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public void validateAndInitializeValue(String valueStr) throws IOException {
+ NodeLabelUtil.checkAndThrowAttributeValue(valueStr);
+ this.value = valueStr;
+ }
+
+ @Override
+ public String getValue() {
+ return value;
+ }
+
+ public String toString() {
+ return getValue();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 8421eea..2615beb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -34,11 +34,6 @@ import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
@@ -64,6 +59,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factories.impl.pb.RecordFactoryPBImpl;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
import org.apache.hadoop.yarn.server.api.ResourceManagerConstants;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.ServerRMProxy;
@@ -76,23 +72,27 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.AppCollectorData;
import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
-import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePlugin;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.ResourcePluginManager;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
@@ -1012,7 +1012,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
StringBuilder errorMsg = new StringBuilder("");
while (iterator.hasNext()) {
try {
- CommonNodeLabelsManager
+ NodeLabelUtil
.checkAndThrowLabelName(iterator.next().getName());
} catch (IOException e) {
errorMsg.append(e.getMessage());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 66065e3..2fe98d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
@@ -99,6 +100,7 @@ public class RMActiveServiceContext {
private ApplicationMasterService applicationMasterService;
private RMNodeLabelsManager nodeLabelManager;
+ private NodeAttributesManager nodeAttributesManager;
private RMDelegatedNodeLabelsUpdater rmDelegatedNodeLabelsUpdater;
private long epoch;
private Clock systemClock = SystemClock.getInstance();
@@ -404,6 +406,18 @@ public class RMActiveServiceContext {
@Private
@Unstable
+ public NodeAttributesManager getNodeAttributesManager() {
+ return nodeAttributesManager;
+ }
+
+ @Private
+ @Unstable
+ public void setNodeAttributesManager(NodeAttributesManager mgr) {
+ nodeAttributesManager = mgr;
+ }
+
+ @Private
+ @Unstable
public AllocationTagsManager getAllocationTagsManager() {
return allocationTagsManager;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index eb91a31..2e2eadf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -132,6 +133,10 @@ public interface RMContext extends ApplicationMasterServiceContext {
public void setNodeLabelManager(RMNodeLabelsManager mgr);
+ NodeAttributesManager getNodeAttributesManager();
+
+ void setNodeAttributesManager(NodeAttributesManager mgr);
+
RMDelegatedNodeLabelsUpdater getRMDelegatedNodeLabelsUpdater();
void setRMDelegatedNodeLabelsUpdater(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 84e0f6f..46930ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -504,6 +505,11 @@ public class RMContextImpl implements RMContext {
}
@Override
+ public void setNodeAttributesManager(NodeAttributesManager mgr) {
+ activeServiceContext.setNodeAttributesManager(mgr);
+ }
+
+ @Override
public AllocationTagsManager getAllocationTagsManager() {
return activeServiceContext.getAllocationTagsManager();
}
@@ -620,4 +626,9 @@ public class RMContextImpl implements RMContext {
this.activeServiceContext.setResourceProfilesManager(mgr);
}
// Note: Read java doc before adding any services over here.
+
+ @Override
+ public NodeAttributesManager getNodeAttributesManager() {
+ return activeServiceContext.getNodeAttributesManager();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 733da5b..8a7503d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -62,15 +62,17 @@ import org.apache.hadoop.yarn.event.EventDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventType;
import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.ApplicationMasterLauncher;
import org.apache.hadoop.yarn.server.resourcemanager.federation.FederationStateStoreService;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.CombinedSystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.NoOpSystemMetricPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV1Publisher;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.TimelineServiceV2Publisher;
-import org.apache.hadoop.yarn.server.resourcemanager.metrics.CombinedSystemMetricsPublisher;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeAttributesManagerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMDelegatedNodeLabelsUpdater;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.NullRMStateStore;
@@ -245,6 +247,13 @@ public class ResourceManager extends CompositeService implements Recoverable {
this.rmContext = new RMContextImpl();
rmContext.setResourceManager(this);
+
+ // add resource profiles here because it's used by AbstractYarnScheduler
+ ResourceProfilesManager resourceProfilesManager =
+ createResourceProfileManager();
+ resourceProfilesManager.init(conf);
+ rmContext.setResourceProfilesManager(resourceProfilesManager);
+
this.configurationProvider =
ConfigurationProviderFactory.getConfigurationProvider(conf);
this.configurationProvider.init(this.conf);
@@ -253,8 +262,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
// load core-site.xml
loadConfigurationXml(YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
- // Refresh user to group mappings during init.
- refreshUserToGroupMappingsWithConf();
+ // Do refreshUserToGroupsMappings with loaded core-site.xml
+ Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(this.conf)
+ .refresh();
// Do refreshSuperUserGroupsConfiguration with loaded core-site.xml
// Or use RM specific configurations to overwrite the common ones first
@@ -339,21 +349,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
super.serviceInit(this.conf);
}
- private void refreshUserToGroupMappingsWithConf()
- throws YarnException, IOException {
- Configuration newConf = new Configuration(false);
- InputStream confFileInputStream =
- configurationProvider
- .getConfigurationInputStream(newConf, YarnConfiguration.CORE_SITE_CONFIGURATION_FILE);
- if (confFileInputStream != null) {
- newConf.addResource(confFileInputStream);
- }
-
- // Do refreshUserToGroupsMappings with loaded core-site.xml
- Groups.getUserToGroupsMappingServiceWithLoadedConfiguration(newConf)
- .refresh();
- }
-
private void loadConfigurationXml(String configurationFile)
throws YarnException, IOException {
InputStream configurationInputStream =
@@ -504,6 +499,11 @@ public class ResourceManager extends CompositeService implements Recoverable {
return new RMNodeLabelsManager();
}
+ protected NodeAttributesManager createNodeAttributesManager()
+ throws InstantiationException, IllegalAccessException {
+ return new NodeAttributesManagerImpl();
+ }
+
protected AllocationTagsManager createAllocationTagsManager() {
return new AllocationTagsManager(this.rmContext);
}
@@ -639,6 +639,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
addService(nlm);
rmContext.setNodeLabelManager(nlm);
+ NodeAttributesManager nam = createNodeAttributesManager();
+ addService(nam);
+ rmContext.setNodeAttributesManager(nam);
+
AllocationTagsManager allocationTagsManager =
createAllocationTagsManager();
rmContext.setAllocationTagsManager(allocationTagsManager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
new file mode 100644
index 0000000..2e63a7c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -0,0 +1,527 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.nodelabels.AttributeValue;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
+import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
+import org.apache.hadoop.yarn.nodelabels.RMNodeAttribute;
+import org.apache.hadoop.yarn.nodelabels.StringAttributeValue;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
+
+/**
+ * Manager holding the attributes to Labels.
+ */
+public class NodeAttributesManagerImpl extends NodeAttributesManager {
+ protected static final Log LOG =
+ LogFactory.getLog(NodeAttributesManagerImpl.class);
+ /**
+ * If a user doesn't specify value for a label, then empty string is
+ * considered as default.
+ */
+ public static final String EMPTY_ATTRIBUTE_VALUE = "";
+
+ private Dispatcher dispatcher;
+
+ // TODO may be we can have a better collection here.
+ // this will be updated to get the attributeName to NM mapping
+ private ConcurrentHashMap<NodeAttribute, RMNodeAttribute> clusterAttributes =
+ new ConcurrentHashMap<>();
+
+ // hostname -> (Map (attributeName -> NodeAttribute))
+ // Instead of NodeAttribute, plan to have it in future as AttributeValue
+ // AttributeValue
+ // / \
+ // StringNodeAttributeValue LongAttributeValue
+ // and convert the configured value to the specific type so that the
+ // expression evaluations are faster
+ private ConcurrentMap<String, Host> nodeCollections =
+ new ConcurrentHashMap<>();
+
+ private final ReadLock readLock;
+ private final WriteLock writeLock;
+
+ public NodeAttributesManagerImpl() {
+ super("NodeAttributesManagerImpl");
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ }
+
+ protected void initDispatcher(Configuration conf) {
+ // create async handler
+ dispatcher = new AsyncDispatcher("AttributeNodeLabelsManager dispatcher");
+ AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
+ asyncDispatcher.init(conf);
+ asyncDispatcher.setDrainEventsOnStop();
+ }
+
+ protected void startDispatcher() {
+ // start dispatcher
+ AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher;
+ asyncDispatcher.start();
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ initNodeAttributeStore(getConfig());
+ // init dispatcher only when service start, because recover will happen in
+ // service init, we don't want to trigger any event handling at that time.
+ initDispatcher(getConfig());
+
+ if (null != dispatcher) {
+ dispatcher.register(NodeAttributesStoreEventType.class,
+ new ForwardingEventHandler());
+ }
+
+ startDispatcher();
+ super.serviceStart();
+ }
+
+ protected void initNodeAttributeStore(Configuration conf) throws Exception {
+ // TODO to generalize and make use of the FileSystemNodeLabelsStore
+ }
+
+ private void internalUpdateAttributesOnNodes(
+ Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
+ AttributeMappingOperationType op,
+ Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded) {
+ try {
+ writeLock.lock();
+
+ // shows node->attributes Mapped as part of this operation.
+ StringBuilder logMsg = new StringBuilder(op.name());
+ logMsg.append(" attributes on nodes:");
+ // do update labels from nodes
+ for (Entry<String, Map<NodeAttribute, AttributeValue>> entry : nodeAttributeMapping
+ .entrySet()) {
+ String nodeHost = entry.getKey();
+ Map<NodeAttribute, AttributeValue> attributes = entry.getValue();
+
+ Host node = nodeCollections.get(nodeHost);
+ if (node == null) {
+ node = new Host(nodeHost);
+ }
+ switch (op) {
+ case REMOVE:
+ removeNodeFromAttributes(nodeHost, attributes.keySet());
+ node.removeAttributes(attributes);
+ break;
+ case ADD:
+ clusterAttributes.putAll(newAttributesToBeAdded);
+ addNodeToAttribute(nodeHost, attributes);
+ node.addAttributes(attributes);
+ break;
+ case REPLACE:
+ clusterAttributes.putAll(newAttributesToBeAdded);
+ replaceNodeToAttribute(nodeHost, node.getAttributes(), attributes);
+ node.replaceAttributes(attributes);
+ break;
+ default:
+ break;
+ }
+ logMsg.append(" NM = ");
+ logMsg.append(entry.getKey());
+ logMsg.append(", attributes=[ ");
+ logMsg.append(StringUtils.join(entry.getValue().entrySet(), ","));
+ logMsg.append("] ,");
+ }
+
+ LOG.info(logMsg);
+
+ if (null != dispatcher) {
+ dispatcher.getEventHandler()
+ .handle(new NodeAttributesStoreEvent(nodeAttributeMapping, op));
+ }
+
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void removeNodeFromAttributes(String nodeHost,
+ Set<NodeAttribute> attributeMappings) {
+ for (NodeAttribute attribute : attributeMappings) {
+ clusterAttributes.get(attribute).removeNode(nodeHost);
+ }
+ }
+
+ private void addNodeToAttribute(String nodeHost,
+ Map<NodeAttribute, AttributeValue> attributeMappings) {
+ for (NodeAttribute attribute : attributeMappings.keySet()) {
+ clusterAttributes.get(attribute).addNode(nodeHost);
+ }
+ }
+
+ private void replaceNodeToAttribute(String nodeHost,
+ Map<NodeAttribute, AttributeValue> oldAttributeMappings,
+ Map<NodeAttribute, AttributeValue> newAttributeMappings) {
+ if (oldAttributeMappings != null) {
+ removeNodeFromAttributes(nodeHost, oldAttributeMappings.keySet());
+ }
+ addNodeToAttribute(nodeHost, newAttributeMappings);
+ }
+
+ /**
+ * @param nodeAttributeMapping
+ * @param newAttributesToBeAdded
+ * @param isRemoveOperation : to indicate whether its a remove operation.
+ * @return Map<String, Map<NodeAttribute, AttributeValue>>, node -> Map(
+ * NodeAttribute -> AttributeValue)
+ * @throws IOException : on invalid mapping in the current request or against
+ * already existing NodeAttributes.
+ */
+ protected Map<String, Map<NodeAttribute, AttributeValue>> validate(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping,
+ Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded,
+ boolean isRemoveOperation) throws IOException {
+ Map<String, Map<NodeAttribute, AttributeValue>> nodeToAttributesMap =
+ new TreeMap<>();
+ Map<NodeAttribute, AttributeValue> attributesValues;
+ Set<Entry<String, Set<NodeAttribute>>> entrySet =
+ nodeAttributeMapping.entrySet();
+ for (Entry<String, Set<NodeAttribute>> nodeToAttrMappingEntry : entrySet) {
+ attributesValues = new HashMap<>();
+ String node = nodeToAttrMappingEntry.getKey().trim();
+ if (nodeToAttrMappingEntry.getValue().isEmpty()) {
+ // no attributes to map mostly remove operation
+ continue;
+ }
+
+ // validate for attributes
+ for (NodeAttribute attribute : nodeToAttrMappingEntry.getValue()) {
+ String attributeName = attribute.getAttributeName().trim();
+ NodeLabelUtil.checkAndThrowLabelName(attributeName);
+ NodeLabelUtil
+ .checkAndThrowAttributePrefix(attribute.getAttributePrefix());
+
+ // ensure trimmed values are set back
+ attribute.setAttributeName(attributeName);
+ attribute.setAttributePrefix(attribute.getAttributePrefix().trim());
+
+ // verify for type against prefix/attributeName
+ if (validateForAttributeTypeMismatch(isRemoveOperation, attribute,
+ newAttributesToBeAdded)) {
+ newAttributesToBeAdded.put(attribute,
+ new RMNodeAttribute(attribute));
+ }
+ // TODO type based value setting needs to be done using a factory
+ StringAttributeValue value = new StringAttributeValue();
+ value.validateAndInitializeValue(
+ normalizeAttributeValue(attribute.getAttributeValue()));
+ attributesValues.put(attribute, value);
+ }
+ nodeToAttributesMap.put(node, attributesValues);
+ }
+ return nodeToAttributesMap;
+ }
+
+ /**
+ *
+ * @param isRemoveOperation
+ * @param attribute
+ * @param newAttributes
+ * @return Whether its a new Attribute added
+ * @throws IOException
+ */
+ private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation,
+ NodeAttribute attribute,
+ Map<NodeAttribute, RMNodeAttribute> newAttributes)
+ throws IOException {
+ if (isRemoveOperation && !clusterAttributes.containsKey(attribute)) {
+ // no need to validate anything as its remove operation and attribute
+ // doesn't exist.
+ return false; // no need to add as its remove operation
+ } else {
+ // already existing or attribute is mapped to another Node in the
+ // current command, then check whether the attribute type is matching
+ NodeAttribute existingAttribute =
+ (clusterAttributes.containsKey((attribute))
+ ? clusterAttributes.get(attribute).getAttribute()
+ : (newAttributes.containsKey(attribute)
+ ? newAttributes.get(attribute).getAttribute()
+ : null));
+ if (existingAttribute == null) {
+ return true;
+ } else if (existingAttribute.getAttributeType() != attribute
+ .getAttributeType()) {
+ throw new IOException("Attribute name - type is not matching with "
+ + "already configured mapping for the attribute "
+ + attribute.getAttributeName() + " existing : "
+ + existingAttribute.getAttributeType() + ", new :"
+ + attribute.getAttributeType());
+ }
+ return false;
+ }
+ }
+
+ protected String normalizeAttributeValue(String value) {
+ if (value != null) {
+ return value.trim();
+ }
+ return EMPTY_ATTRIBUTE_VALUE;
+ }
+
+ @Override
+ public Set<NodeAttribute> getClusterNodeAttributes(Set<String> prefix) {
+ Set<NodeAttribute> attributes = new HashSet<>();
+ try {
+ readLock.lock();
+ attributes.addAll(clusterAttributes.keySet());
+ } finally {
+ readLock.unlock();
+ }
+ if (prefix != null && prefix.isEmpty()) {
+ Iterator<NodeAttribute> iterator = attributes.iterator();
+ while (iterator.hasNext()) {
+ NodeAttribute attribute = iterator.next();
+ if (!prefix.contains(attribute.getAttributePrefix())) {
+ iterator.remove();
+ }
+ }
+ }
+ return attributes;
+ }
+
+ // TODO need to handle as part of REST patch.
+ /*
+ * @Override public Map<NodeAttribute, Set<String>> getAttributesToNodes(
+ * Set<NodeAttribute> attributes) { try { readLock.lock(); boolean
+ * fetchAllAttributes = (attributes == null || attributes.isEmpty());
+ * Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>(); for
+ * (Entry<NodeAttribute, RMAttributeNodeLabel> attributeEntry :
+ * attributeCollections .entrySet()) { if (fetchAllAttributes ||
+ * attributes.contains(attributeEntry.getKey())) {
+ * attributesToNodes.put(attributeEntry.getKey(),
+ * attributeEntry.getValue().getAssociatedNodeIds()); } } return
+ * attributesToNodes; } finally { readLock.unlock(); } }
+ */
+
+ public Resource getResourceByAttribute(NodeAttribute attribute) {
+ try {
+ readLock.lock();
+ return clusterAttributes.containsKey(attribute)
+ ? clusterAttributes.get(attribute).getResource()
+ : Resource.newInstance(0, 0);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public Map<NodeAttribute, AttributeValue> getAttributesForNode(
+ String hostName) {
+ try {
+ readLock.lock();
+ return nodeCollections.containsKey(hostName)
+ ? nodeCollections.get(hostName).getAttributes()
+ : new HashMap<>();
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public void activateNode(NodeId nodeId, Resource resource) {
+ try {
+ writeLock.lock();
+ String hostName = nodeId.getHost();
+ Host host = nodeCollections.get(hostName);
+ if (host == null) {
+ host = new Host(hostName);
+ nodeCollections.put(hostName, host);
+ }
+ host.activateNode(resource);
+ for (NodeAttribute attribute : host.getAttributes().keySet()) {
+ clusterAttributes.get(attribute).removeNode(resource);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void deactivateNode(NodeId nodeId) {
+ try {
+ writeLock.lock();
+ Host host = nodeCollections.get(nodeId.getHost());
+ for (NodeAttribute attribute : host.getAttributes().keySet()) {
+ clusterAttributes.get(attribute).removeNode(host.getResource());
+ }
+ host.deactivateNode();
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public void updateNodeResource(NodeId node, Resource newResource) {
+ deactivateNode(node);
+ activateNode(node, newResource);
+ }
+
+ /**
+ * A <code>Host</code> can have multiple <code>Node</code>s.
+ */
+ public static class Host {
+ private String hostName;
+ private Map<NodeAttribute, AttributeValue> attributes;
+ private Resource resource;
+ private boolean isActive;
+
+ private Map<NodeAttribute, AttributeValue> getAttributes() {
+ return attributes;
+ }
+
+ public void setAttributes(Map<NodeAttribute, AttributeValue> attributes) {
+ this.attributes = attributes;
+ }
+
+ public void removeAttributes(
+ Map<NodeAttribute, AttributeValue> attributesMapping) {
+ for (NodeAttribute attribute : attributesMapping.keySet()) {
+ this.attributes.remove(attribute);
+ }
+ }
+
+ public void replaceAttributes(
+ Map<NodeAttribute, AttributeValue> attributesMapping) {
+ this.attributes.clear();
+ this.attributes.putAll(attributesMapping);
+ }
+
+ public void addAttributes(
+ Map<NodeAttribute, AttributeValue> attributesMapping) {
+ this.attributes.putAll(attributesMapping);
+ }
+
+ public Resource getResource() {
+ return resource;
+ }
+
+ public void setResource(Resource resourceParam) {
+ this.resource = resourceParam;
+ }
+
+ public boolean isActive() {
+ return isActive;
+ }
+
+ public void deactivateNode() {
+ this.isActive = false;
+ this.resource = Resource.newInstance(0, 0);
+ }
+
+ public void activateNode(Resource r) {
+ this.isActive = true;
+ this.resource = r;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ public void setHostName(String hostName) {
+ this.hostName = hostName;
+ }
+
+ public Host(String hostName) {
+ this(hostName, new HashMap<NodeAttribute, AttributeValue>());
+ }
+
+ public Host(String hostName,
+ Map<NodeAttribute, AttributeValue> attributes) {
+ this(hostName, attributes, Resource.newInstance(0, 0), false);
+ }
+
+ public Host(String hostName, Map<NodeAttribute, AttributeValue> attributes,
+ Resource resource, boolean isActive) {
+ super();
+ this.attributes = attributes;
+ this.resource = resource;
+ this.isActive = isActive;
+ this.hostName = hostName;
+ }
+ }
+
+ private final class ForwardingEventHandler
+ implements EventHandler<NodeAttributesStoreEvent> {
+
+ @Override
+ public void handle(NodeAttributesStoreEvent event) {
+ handleStoreEvent(event);
+ }
+ }
+
+ // Dispatcher related code
+ protected void handleStoreEvent(NodeAttributesStoreEvent event) {
+ // TODO Need to extend the File
+ }
+
+ @Override
+ public void replaceNodeAttributes(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException {
+ processMapping(nodeAttributeMapping, AttributeMappingOperationType.REPLACE);
+ }
+
+ @Override
+ public void addNodeAttributes(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException {
+ processMapping(nodeAttributeMapping, AttributeMappingOperationType.ADD);
+ }
+
+ @Override
+ public void removeNodeAttributes(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping) throws IOException {
+ processMapping(nodeAttributeMapping, AttributeMappingOperationType.REMOVE);
+ }
+
+ private void processMapping(
+ Map<String, Set<NodeAttribute>> nodeAttributeMapping,
+ AttributeMappingOperationType mappingType) throws IOException {
+ Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded =
+ new HashMap<>();
+ Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
+ validate(nodeAttributeMapping, newAttributesToBeAdded, false);
+
+ internalUpdateAttributesOnNodes(validMapping, mappingType,
+ newAttributesToBeAdded);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java
new file mode 100644
index 0000000..d04e8cf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEvent.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.event.AbstractEvent;
+import org.apache.hadoop.yarn.nodelabels.AttributeValue;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
+
+/**
+ * Event capturing details to store the Node Attributes in the backend store.
+ */
+public class NodeAttributesStoreEvent
+ extends AbstractEvent<NodeAttributesStoreEventType> {
+ private Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping;
+ private AttributeMappingOperationType operation;
+
+ public NodeAttributesStoreEvent(
+ Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMappingList,
+ AttributeMappingOperationType operation) {
+ super(NodeAttributesStoreEventType.STORE_ATTRIBUTES);
+ this.nodeAttributeMapping = nodeAttributeMappingList;
+ this.operation = operation;
+ }
+
+ public Map<String, Map<NodeAttribute, AttributeValue>> getNodeAttributeMappingList() {
+ return nodeAttributeMapping;
+ }
+
+ public AttributeMappingOperationType getOperation() {
+ return operation;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/06fbea4e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEventType.java
new file mode 100644
index 0000000..c433b19
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesStoreEventType.java
@@ -0,0 +1,26 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+
+/**
+ * Event type to store the NodeAttributes.
+ */
+public enum NodeAttributesStoreEventType {
+ STORE_ATTRIBUTES
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-8068. Application Priority
field causes NPE in app timeline publish when Hadoop 2.7 based clients to
2.8+ (Sunil G via wangda)
Posted by na...@apache.org.
YARN-8068. Application Priority field causes NPE in app timeline publish when Hadoop 2.7 based clients to 2.8+ (Sunil G via wangda)
Change-Id: I7910bd1064a1b4dbbe2084080c060822ea6f3b48
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9eef19b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9eef19b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9eef19b2
Branch: refs/heads/YARN-3409
Commit: 9eef19b2ad78b8464da252d0e23c08675898b9d8
Parents: 5d38157
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Mar 26 11:19:15 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Mar 26 11:19:15 2018 -0700
----------------------------------------------------------------------
.../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9eef19b2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 61936bb..4234e58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -455,6 +455,11 @@ public class RMAppImpl implements RMApp, Recoverable {
if (submissionContext.getPriority() != null) {
this.applicationPriority = Priority
.newInstance(submissionContext.getPriority().getPriority());
+ } else {
+ // If incoming app does not have priority configured in submission
+ // context, system could be assume that its a 0 priority app and could be
+ // considered as normal.
+ this.applicationPriority = Priority.newInstance(0);
}
int globalMaxAppAttempts = conf.getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-6257. Fix CapacityScheduler
REST API JSON output. Contributed By Tao Yang
Posted by na...@apache.org.
YARN-6257. Fix CapacityScheduler REST API JSON output.
Contributed By Tao Yang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fb00c3f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fb00c3f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fb00c3f
Branch: refs/heads/YARN-3409
Commit: 8fb00c3fce96b445a71fb1a5d98b6a55d6ebfd10
Parents: 9d7a903
Author: Eric Yang <ey...@apache.org>
Authored: Thu Mar 29 17:36:34 2018 -0400
Committer: Eric Yang <ey...@apache.org>
Committed: Thu Mar 29 17:36:34 2018 -0400
----------------------------------------------------------------------
.../webapp/dao/CapacitySchedulerHealthInfo.java | 35 +++--
.../webapp/TestRMWebServicesCapacitySched.java | 4 +
.../src/site/markdown/ResourceManagerRest.md | 132 +++++++++++++++++++
3 files changed, 158 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
index 652c91b..3683055 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerHealthInfo.java
@@ -25,15 +25,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
@XmlAccessorType(XmlAccessType.FIELD)
public class CapacitySchedulerHealthInfo {
@XmlAccessorType(XmlAccessType.FIELD)
public static class OperationInformation {
+ String operation;
String nodeId;
String containerId;
String queue;
@@ -41,7 +40,9 @@ public class CapacitySchedulerHealthInfo {
OperationInformation() {
}
- OperationInformation(SchedulerHealth.DetailedInformation di) {
+ OperationInformation(String operation,
+ SchedulerHealth.DetailedInformation di) {
+ this.operation = operation;
this.nodeId = di.getNodeId() == null ? "N/A" : di.getNodeId().toString();
this.containerId =
di.getContainerId() == null ? "N/A" : di.getContainerId().toString();
@@ -59,6 +60,10 @@ public class CapacitySchedulerHealthInfo {
public String getQueue() {
return queue;
}
+
+ public String getOperation() {
+ return operation;
+ }
}
@XmlAccessorType(XmlAccessType.FIELD)
@@ -90,7 +95,7 @@ public class CapacitySchedulerHealthInfo {
}
long lastrun;
- Map<String, OperationInformation> operationsInfo;
+ List<OperationInformation> operationsInfo;
List<LastRunDetails> lastRunDetails;
CapacitySchedulerHealthInfo() {
@@ -100,18 +105,22 @@ public class CapacitySchedulerHealthInfo {
return lastrun;
}
+ public List<OperationInformation> getOperationsInfo() {
+ return operationsInfo;
+ }
+
CapacitySchedulerHealthInfo(CapacityScheduler cs) {
SchedulerHealth ht = cs.getSchedulerHealth();
lastrun = ht.getLastSchedulerRunTime();
- operationsInfo = new HashMap<>();
- operationsInfo.put("last-allocation",
- new OperationInformation(ht.getLastAllocationDetails()));
- operationsInfo.put("last-release",
- new OperationInformation(ht.getLastReleaseDetails()));
- operationsInfo.put("last-preemption",
- new OperationInformation(ht.getLastPreemptionDetails()));
- operationsInfo.put("last-reservation",
- new OperationInformation(ht.getLastReservationDetails()));
+ operationsInfo = new ArrayList<>();
+ operationsInfo.add(new OperationInformation("last-allocation",
+ ht.getLastAllocationDetails()));
+ operationsInfo.add(
+ new OperationInformation("last-release", ht.getLastReleaseDetails()));
+ operationsInfo.add(new OperationInformation("last-preemption",
+ ht.getLastPreemptionDetails()));
+ operationsInfo.add(new OperationInformation("last-reservation",
+ ht.getLastReservationDetails()));
lastRunDetails = new ArrayList<>();
lastRunDetails.add(new LastRunDetails("releases", ht.getReleaseCount(), ht
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 86a3943..e37f76f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -329,6 +329,10 @@ public class TestRMWebServicesCapacitySched extends JerseyTestBase {
JSONObject health = info.getJSONObject("health");
assertNotNull(health);
assertEquals("incorrect number of elements", 3, health.length());
+ JSONArray operationsInfo = health.getJSONArray("operationsInfo");
+ assertEquals("incorrect number of elements", 4, operationsInfo.length());
+ JSONArray lastRunDetails = health.getJSONArray("lastRunDetails");
+ assertEquals("incorrect number of elements", 3, lastRunDetails.length());
JSONArray arr = info.getJSONObject("queues").getJSONArray("queue");
assertEquals("incorrect number of elements", 2, arr.length());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fb00c3f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
index c43fe14..caeaf3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/ResourceManagerRest.md
@@ -311,6 +311,7 @@ The capacity scheduler supports hierarchical queues. This one request will print
| maxCapacity | float | Configured maximum queue capacity in percentage relative to its parent queue |
| queueName | string | Name of the queue |
| queues | array of queues(JSON)/zero or more queue objects(XML) | A collection of queue resources |
+| health | A single health object | The health metrics of capacity scheduler. This metrics existed since 2.8.0, but the output was not well formatted. Hence users can not make use of this field cleanly, this is optimized from 3.2.0 onwards. |
### Elements of the queues object for a Parent queue
@@ -361,6 +362,31 @@ The capacity scheduler supports hierarchical queues. This one request will print
| memory | int | The amount of memory used (in MB) |
| vCores | int | The number of virtual cores |
+### Elements of the health object in schedulerInfo:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| lastrun | long | The time in which application started (in ms since epoch) |
+| operationsInfo | array of operations(JSON)/operation objects(XML) | A collection of operation objects |
+| lastRunDetails | array of lastRunDetails(JSON)/lastRunDetail objects(XML) | A collection of lastRunDetail objects |
+
+### Elements of the operation object in health:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| operation | string | The type of operation |
+| nodeId | string | The id of the node to which the operation relates |
+| containerId | string | The id of the container to which the operation relates |
+| queue | string | The name of the queue to which the operation relates |
+
+### Elements of the lastRunDetail object in health:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| operation | string | The type of operation |
+| count | long | The id of the node to which the operation relates |
+| resources | A single resource object | The resources to which the operation relates |
+
#### Response Examples
**JSON response**
@@ -632,6 +658,61 @@ Response Body:
}
]
},
+ "health": {
+ "lastrun": 1326381444693,
+ "operationsInfo": [
+ {
+ "operation": "last-allocation",
+ "nodeId": "N/A",
+ "containerId": "N/A",
+ "queue": "N/A"
+ },
+ {
+ "operation": "last-release",
+ "nodeId": "host.domain.com:8041",
+ "containerId": "container_1326821518301_0005_01_000001",
+ "queue": "root.default"
+ },
+ {
+ "operation": "last-preemption",
+ "nodeId": "N/A",
+ "containerId": "N/A",
+ "queue": "N/A"
+ },
+ {
+ "operation": "last-reservation",
+ "nodeId": "N/A",
+ "containerId": "N/A",
+ "queue": "N/A"
+ }
+ ],
+ "lastRunDetails": [
+ {
+ "operation": "releases",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ },
+ {
+ "operation": "allocations",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ },
+ {
+ "operation": "reservations",
+ "count": 0,
+ "resources": {
+ "memory": 0,
+ "vCores": 0
+ }
+ }
+ ]
+ },
"type": "capacityScheduler",
"usedCapacity": 0.0
}
@@ -894,6 +975,57 @@ Response Body:
</resourcesUsed>
</queue>
</queues>
+ <health>
+ <lastrun>1326381444693</lastrun>
+ <operationsInfo>
+ <operation>last-allocation</operation>
+ <nodeId>N/A</nodeId>
+ <containerId>N/A</containerId>
+ <queue>N/A</queue>
+ </operationsInfo>
+ <operationsInfo>
+ <operation>last-release</operation>
+ <nodeId>host.domain.com:8041</nodeId>
+ <containerId>container_1326821518301_0005_01_000001</containerId>
+ <queue>root.default</queue>
+ </operationsInfo>
+ <operationsInfo>
+ <operation>last-preemption</operation>
+ <nodeId>N/A</nodeId>
+ <containerId>N/A</containerId>
+ <queue>N/A</queue>
+ </operationsInfo>
+ <operationsInfo>
+ <operation>last-reservation</operation>
+ <nodeId>N/A</nodeId>
+ <containerId>N/A</containerId>
+ <queue>N/A</queue>
+ </operationsInfo>
+ <lastRunDetails>
+ <operation>releases</operation>
+ <count>0</count>
+ <resources>
+ <memory>0</memory>
+ <vCores>0</vCores>
+ </resources>
+ </lastRunDetails>
+ <lastRunDetails>
+ <operation>allocations</operation>
+ <count>0</count>
+ <resources>
+ <memory>0</memory>
+ <vCores>0</vCores>
+ </resources>
+ </lastRunDetails>
+ <lastRunDetails>
+ <operation>reservations</operation>
+ <count>0</count>
+ <resources>
+ <memory>0</memory>
+ <vCores>0</vCores>
+ </resources>
+ </lastRunDetails>
+ </health>
</schedulerInfo>
</scheduler>
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: Revert "YARN-8010. Add config in
FederationRMFailoverProxy to not bypass facade cache when failing over.
(Botong Huang via Subru)."
Posted by na...@apache.org.
Revert "YARN-8010. Add config in FederationRMFailoverProxy to not bypass facade cache when failing over. (Botong Huang via Subru)."
This reverts commit 2a2ef15caf791f30c471526c1b74e68803f0c405 as smart-apply-patch script didn't pick the latest patch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/725b10e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/725b10e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/725b10e3
Branch: refs/heads/YARN-3409
Commit: 725b10e3aee383d049c97f8ed2b0b1ae873d5ae8
Parents: 47f711e
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Mar 28 11:26:50 2018 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Mar 28 11:26:50 2018 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/yarn/conf/YarnConfiguration.java | 9 +++------
.../src/main/resources/yarn-default.xml | 9 ---------
.../failover/FederationRMFailoverProxyProvider.java | 11 +++--------
3 files changed, 6 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/725b10e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 658099a..04b2898 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -3089,18 +3089,15 @@ public class YarnConfiguration extends Configuration {
public static final String FEDERATION_CACHE_TIME_TO_LIVE_SECS =
FEDERATION_PREFIX + "cache-ttl.secs";
- // 5 minutes
- public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
-
- public static final String FEDERATION_FLUSh_CACHE_FOR_RM_ADDR =
- FEDERATION_PREFIX + "flush-cache-for-rm-addr";
- public static final boolean DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR = true;
public static final String FEDERATION_REGISTRY_BASE_KEY =
FEDERATION_PREFIX + "registry.base-dir";
public static final String DEFAULT_FEDERATION_REGISTRY_BASE_KEY =
"yarnfederation/";
+ // 5 minutes
+ public static final int DEFAULT_FEDERATION_CACHE_TIME_TO_LIVE_SECS = 5 * 60;
+
public static final String FEDERATION_STATESTORE_HEARTBEAT_INTERVAL_SECS =
FEDERATION_PREFIX + "state-store.heartbeat-interval-secs";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/725b10e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2eba8df..114ba4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2925,15 +2925,6 @@
</property>
<property>
- <description>
- Whether to flush FederationStateStoreFacade cache to get subcluster info
- when FederationRMFailoverProxyProvider is performing failover.
- </description>
- <name>yarn.federation.flush-cache-for-rm-addr</name>
- <value>true</value>
- </property>
-
- <property>
<description>The registry base directory for federation.</description>
<name>yarn.federation.registry.base-dir</name>
<value>yarnfederation/</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/725b10e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
index b72b199..c631208 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/failover/FederationRMFailoverProxyProvider.java
@@ -64,8 +64,7 @@ public class FederationRMFailoverProxyProvider<T>
private FederationStateStoreFacade facade;
private SubClusterId subClusterId;
private UserGroupInformation originalUser;
- private boolean federationFailoverEnabled;
- private boolean flushFacadeCacheForYarnRMAddr;
+ private boolean federationFailoverEnabled = false;
@Override
public void init(Configuration configuration, RMProxy<T> proxy,
@@ -76,16 +75,13 @@ public class FederationRMFailoverProxyProvider<T>
String clusterId = configuration.get(YarnConfiguration.RM_CLUSTER_ID);
Preconditions.checkNotNull(clusterId, "Missing RM ClusterId");
this.subClusterId = SubClusterId.newInstance(clusterId);
- this.facade = FederationStateStoreFacade.getInstance();
+ this.facade = facade.getInstance();
if (configuration instanceof YarnConfiguration) {
this.conf = (YarnConfiguration) configuration;
}
federationFailoverEnabled =
conf.getBoolean(YarnConfiguration.FEDERATION_FAILOVER_ENABLED,
YarnConfiguration.DEFAULT_FEDERATION_FAILOVER_ENABLED);
- flushFacadeCacheForYarnRMAddr =
- conf.getBoolean(YarnConfiguration.FEDERATION_FLUSh_CACHE_FOR_RM_ADDR,
- YarnConfiguration.DEFAULT_FEDERATION_FLUSh_CACHE_FOR_RM_ADDR);
conf.setInt(
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY,
@@ -123,8 +119,7 @@ public class FederationRMFailoverProxyProvider<T>
try {
LOG.info("Failing over to the ResourceManager for SubClusterId: {}",
subClusterId);
- subClusterInfo = facade.getSubCluster(subClusterId,
- this.flushFacadeCacheForYarnRMAddr && isFailover);
+ subClusterInfo = facade.getSubCluster(subClusterId, isFailover);
// updating the conf with the refreshed RM addresses as proxy
// creations are based out of conf
updateRMAddress(subClusterInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-6629. NPE occurred when
container allocation proposal is applied but its resource requests are
removed before. (Tao Yang via wangda)
Posted by na...@apache.org.
YARN-6629. NPE occurred when container allocation proposal is applied but its resource requests are removed before. (Tao Yang via wangda)
Change-Id: I805880f90b3f6798ec96ed8e8e75755f390a9ad5
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47f711ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47f711ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47f711ee
Branch: refs/heads/YARN-3409
Commit: 47f711eebca315804c80012eea5f31275ac25518
Parents: cdee0a4
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Mar 28 08:47:31 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Mar 28 11:07:45 2018 -0700
----------------------------------------------------------------------
.../scheduler/capacity/CapacityScheduler.java | 4 +-
.../scheduler/common/fica/FiCaSchedulerApp.java | 16 ++++--
.../capacity/TestCapacityScheduler.java | 53 ++++++++++++++++++++
3 files changed, 68 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f711ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index daf0354..bf674a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -2825,8 +2825,8 @@ public class CapacityScheduler extends
// proposal might be outdated if AM failover just finished
// and proposal queue was not be consumed in time
if (app != null && attemptId.equals(app.getApplicationAttemptId())) {
- if (app.accept(cluster, request, updatePending)) {
- app.apply(cluster, request, updatePending);
+ if (app.accept(cluster, request, updatePending)
+ && app.apply(cluster, request, updatePending)) {
LOG.info("Allocation proposal accepted");
isSuccess = true;
} else{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f711ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index f3da0a3..32b2cad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -489,7 +489,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
return accepted;
}
- public void apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,
+ public boolean apply(Resource cluster, ResourceCommitRequest<FiCaSchedulerApp,
FiCaSchedulerNode> request, boolean updatePending) {
boolean reReservation = false;
@@ -502,8 +502,16 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
allocation = request.getFirstAllocatedOrReservedContainer();
SchedulerContainer<FiCaSchedulerApp, FiCaSchedulerNode>
schedulerContainer = allocation.getAllocatedOrReservedContainer();
- RMContainer rmContainer = schedulerContainer.getRmContainer();
+ // Required sanity check - AM can call 'allocate' to update resource
+ // request without locking the scheduler, hence we need to check
+ if (updatePending &&
+ getOutstandingAsksCount(schedulerContainer.getSchedulerRequestKey())
+ <= 0) {
+ return false;
+ }
+
+ RMContainer rmContainer = schedulerContainer.getRmContainer();
reReservation =
(!schedulerContainer.isAllocated()) && (rmContainer.getState()
== RMContainerState.RESERVED);
@@ -545,7 +553,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
containerRequest);
// If this is from a SchedulingRequest, set allocation tags.
- if (containerRequest.getSchedulingRequest() != null) {
+ if (containerRequest != null
+ && containerRequest.getSchedulingRequest() != null) {
((RMContainerImpl) rmContainer).setAllocationTags(
containerRequest.getSchedulingRequest().getAllocationTags());
}
@@ -598,6 +607,7 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
if (!reReservation) {
getCSLeafQueue().apply(cluster, request);
}
+ return true;
}
public boolean unreserve(SchedulerRequestKey schedulerKey,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/47f711ee/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 0fa5c15..1d2aadc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -134,6 +134,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNodeReport;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.TestSchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.ResourceCommitRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -170,6 +171,8 @@ import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
public class TestCapacityScheduler extends CapacitySchedulerTestBase {
private static final Log LOG = LogFactory.getLog(TestCapacityScheduler.class);
@@ -4857,4 +4860,54 @@ public class TestCapacityScheduler extends CapacitySchedulerTestBase {
}
}
}
+
+ @Test (timeout = 60000)
+ public void testClearRequestsBeforeApplyTheProposal()
+ throws Exception {
+ // init RM & NMs & Nodes
+ final MockRM rm = new MockRM(new CapacitySchedulerConfiguration());
+ rm.start();
+ final MockNM nm = rm.registerNode("h1:1234", 200 * GB);
+
+ // submit app
+ final RMApp app = rm.submitApp(200, "app", "user");
+ MockRM.launchAndRegisterAM(app, rm, nm);
+
+ // spy capacity scheduler to handle CapacityScheduler#apply
+ final Priority priority = Priority.newInstance(1);
+ final CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
+ final CapacityScheduler spyCs = Mockito.spy(cs);
+ Mockito.doAnswer(new Answer<Object>() {
+ public Object answer(InvocationOnMock invocation) throws Exception {
+ // clear resource request before applying the proposal for container_2
+ spyCs.allocate(app.getCurrentAppAttempt().getAppAttemptId(),
+ Arrays.asList(ResourceRequest.newInstance(priority, "*",
+ Resources.createResource(1 * GB), 0)), null,
+ Collections.<ContainerId>emptyList(), null, null,
+ NULL_UPDATE_REQUESTS);
+ // trigger real apply which can raise NPE before YARN-6629
+ try {
+ FiCaSchedulerApp schedulerApp = cs.getApplicationAttempt(
+ app.getCurrentAppAttempt().getAppAttemptId());
+ schedulerApp.apply((Resource) invocation.getArguments()[0],
+ (ResourceCommitRequest) invocation.getArguments()[1],
+ (Boolean) invocation.getArguments()[2]);
+ // the proposal of removed request should be rejected
+ Assert.assertEquals(1, schedulerApp.getLiveContainers().size());
+ } catch (Throwable e) {
+ Assert.fail();
+ }
+ return null;
+ }
+ }).when(spyCs).tryCommit(Mockito.any(Resource.class),
+ Mockito.any(ResourceCommitRequest.class), Mockito.anyBoolean());
+
+ // rm allocates container_2 to reproduce the process that can raise NPE
+ spyCs.allocate(app.getCurrentAppAttempt().getAppAttemptId(),
+ Arrays.asList(ResourceRequest.newInstance(priority, "*",
+ Resources.createResource(1 * GB), 1)), null,
+ Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
+ spyCs.handle(new NodeUpdateSchedulerEvent(
+ spyCs.getNode(nm.getNodeId()).getRMNode()));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: HADOOP-15313. TestKMS should close
providers.
Posted by na...@apache.org.
HADOOP-15313. TestKMS should close providers.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c22d62b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c22d62b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c22d62b3
Branch: refs/heads/YARN-3409
Commit: c22d62b338cb16d93c4576a9c634041e3610a116
Parents: 27d60a1
Author: Xiao Chen <xi...@apache.org>
Authored: Mon Mar 26 15:59:17 2018 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Mon Mar 26 15:59:32 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/io/MultipleIOException.java | 10 ++++++
.../hadoop/crypto/key/kms/server/TestKMS.java | 38 ++++++++++++++++----
2 files changed, 41 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22d62b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
index 66c1ab1..c9d7ade 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/MultipleIOException.java
@@ -76,5 +76,15 @@ public class MultipleIOException extends IOException {
public IOException build() {
return createIOException(exceptions);
}
+
+ /**
+ * @return whether any exception was added.
+ */
+ public boolean isEmpty() {
+ if (exceptions == null) {
+ return true;
+ }
+ return exceptions.isEmpty();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c22d62b3/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index 1189fbf..1517b04 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.crypto.key.kms.LoadBalancingKMSClientProvider;
import org.apache.hadoop.crypto.key.kms.ValueQueue;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.SecurityUtil;
@@ -84,6 +85,7 @@ import java.util.Collection;
import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
@@ -111,6 +113,10 @@ public class TestKMS {
private SSLFactory sslFactory;
+ // Keep track of all key providers created during a test case, so they can be
+ // closed at test tearDown.
+ private List<KeyProvider> providersCreated = new LinkedList<>();
+
@Rule
public final Timeout testTimeout = new Timeout(180000);
@@ -144,13 +150,17 @@ public class TestKMS {
protected KeyProvider createProvider(URI uri, Configuration conf)
throws IOException {
- return new LoadBalancingKMSClientProvider(
- new KMSClientProvider[] { new KMSClientProvider(uri, conf) }, conf);
+ final KeyProvider ret = new LoadBalancingKMSClientProvider(
+ new KMSClientProvider[] {new KMSClientProvider(uri, conf)}, conf);
+ providersCreated.add(ret);
+ return ret;
}
private KMSClientProvider createKMSClientProvider(URI uri, Configuration conf)
throws IOException {
- return new KMSClientProvider(uri, conf);
+ final KMSClientProvider ret = new KMSClientProvider(uri, conf);
+ providersCreated.add(ret);
+ return ret;
}
protected <T> T runServer(String keystore, String password, File confDir,
@@ -311,13 +321,28 @@ public class TestKMS {
}
@After
- public void tearDownMiniKdc() throws Exception {
+ public void tearDown() throws Exception {
if (kdc != null) {
kdc.stop();
kdc = null;
}
UserGroupInformation.setShouldRenewImmediatelyForTests(false);
UserGroupInformation.reset();
+ if (!providersCreated.isEmpty()) {
+ final MultipleIOException.Builder b = new MultipleIOException.Builder();
+ for (KeyProvider kp : providersCreated) {
+ try {
+ kp.close();
+ } catch (IOException e) {
+ LOG.error("Failed to close key provider.", e);
+ b.add(e);
+ }
+ }
+ providersCreated.clear();
+ if (!b.isEmpty()) {
+ throw b.build();
+ }
+ }
}
private <T> T doAs(String user, final PrivilegedExceptionAction<T> action)
@@ -449,6 +474,8 @@ public class TestKMS {
}
}
Assert.assertTrue("Reloader is not alive", reloaderThread.isAlive());
+ // Explicitly close the provider so we can verify the internal thread
+ // is shutdown
testKp.close();
boolean reloaderStillAlive = true;
for (int i = 0; i < 10; i++) {
@@ -476,7 +503,6 @@ public class TestKMS {
.addDelegationTokens("myuser", new Credentials());
Assert.assertEquals(1, tokens.length);
Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
- kp.close();
return null;
}
});
@@ -494,7 +520,6 @@ public class TestKMS {
.addDelegationTokens("myuser", new Credentials());
Assert.assertEquals(1, tokens.length);
Assert.assertEquals("kms-dt", tokens[0].getKind().toString());
- kp.close();
}
return null;
}
@@ -2533,7 +2558,6 @@ public class TestKMS {
@Test
public void testTGTRenewal() throws Exception {
- tearDownMiniKdc();
Properties kdcConf = MiniKdc.createConf();
kdcConf.setProperty(MiniKdc.MAX_TICKET_LIFETIME, "3");
kdcConf.setProperty(MiniKdc.MIN_TICKET_LIFETIME, "3");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org