You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2018/04/11 00:37:59 UTC
[1/2] hadoop git commit: YARN-8116. Nodemanager fails with
NumberFormatException: For input string: . (Chandni Singh via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk c467f311d -> d919eb6ef
YARN-8116. Nodemanager fails with NumberFormatException: For input string: . (Chandni Singh via wangda)
Change-Id: Idd30cfca59982d3fc6e47aa1b88f844a78fae94d
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bf9cc2c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bf9cc2c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bf9cc2c
Branch: refs/heads/trunk
Commit: 2bf9cc2c73944c9f7cde56714b8cf6995cfa539b
Parents: c467f31
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Apr 10 17:32:38 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Apr 10 17:32:38 2018 -0700
----------------------------------------------------------------------
.../containermanager/container/ContainerImpl.java | 3 ++-
.../recovery/NMLeveldbStateStoreService.java | 4 +++-
.../recovery/TestNMLeveldbStateStoreService.java | 16 ++++++++++++++++
3 files changed, 21 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 2115100..c09c7f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -2191,7 +2191,8 @@ public class ContainerImpl implements Container {
}
private void storeRetryContext() {
- if (windowRetryContext.getRestartTimes() != null) {
+ if (windowRetryContext.getRestartTimes() != null &&
+ !windowRetryContext.getRestartTimes().isEmpty()) {
try {
stateStore.storeContainerRestartTimes(containerId,
windowRetryContext.getRestartTimes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index bf4c0ad..723dd48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -347,7 +347,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
value.substring(1, value.length() - 1).split(", ");
List<Long> restartTimes = new ArrayList<>();
for (String restartTime : unparsedRestartTimes) {
- restartTimes.add(Long.parseLong(restartTime));
+ if (!restartTime.isEmpty()) {
+ restartTimes.add(Long.parseLong(restartTime));
+ }
}
rcs.setRestartTimes(restartTimes);
} else if (suffix.equals(CONTAINER_WORK_DIR_KEY_SUFFIX)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bf9cc2c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index c270199..265b3e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -1216,6 +1216,22 @@ public class TestNMLeveldbStateStoreService {
Assert.fail("Expected exception not thrown");
}
+ @Test
+ public void testEmptyRestartTimes() throws IOException {
+ List<Long> restartTimes = new ArrayList<>();
+ ApplicationId appId = ApplicationId.newInstance(1234, 3);
+ ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId,
+ 4);
+ ContainerId containerId = ContainerId.newContainerId(appAttemptId, 5);
+ storeMockContainer(containerId);
+ stateStore.storeContainerRestartTimes(containerId,
+ restartTimes);
+ restartStateStore();
+ RecoveredContainerState rcs = stateStore.loadContainersState().get(0);
+ List<Long> recoveredRestartTimes = rcs.getRestartTimes();
+ assertTrue(recoveredRestartTimes.isEmpty());
+ }
+
private StartContainerRequest storeMockContainer(ContainerId containerId)
throws IOException {
// create a container request
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-8133. Doc link broken for yarn-service
from overview page. (Rohith Sharma K S via wangda)
Posted by wa...@apache.org.
YARN-8133. Doc link broken for yarn-service from overview page. (Rohith Sharma K S via wangda)
Change-Id: Iacf9a004585dd59e1c0cd8f8c618a38f047cc0fe
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d919eb6e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d919eb6e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d919eb6e
Branch: refs/heads/trunk
Commit: d919eb6efa1072517017c75fb323e391f4418dc8
Parents: 2bf9cc2
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Apr 10 17:33:15 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Apr 10 17:33:15 2018 -0700
----------------------------------------------------------------------
.../src/site/markdown/yarn-service/Concepts.md | 6 +++---
.../src/site/markdown/yarn-service/Overview.md | 14 +++++++-------
.../src/site/markdown/yarn-service/QuickStart.md | 12 ++++++------
.../src/site/markdown/yarn-service/RegistryDNS.md | 4 ++--
.../site/markdown/yarn-service/ServiceDiscovery.md | 4 ++--
5 files changed, 20 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
index e567d03..5c77f17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Concepts.md
@@ -29,7 +29,7 @@ affinity and anti-affinity scheduling, log aggregation for services, automatical
A restful API server is developed to allow users to deploy/manage their services on YARN via a simple JSON spec. This avoids users
from dealing with the low-level APIs, writing complex code to bring their services onto YARN. The REST layer acts as a unified REST based entry for
creation and lifecycle management of YARN services. Services here can range from simple single-component apps to the most complex,
-multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.md) for detailed API documentations.
+multi-component applications needing special orchestration needs. Please refer to this [API doc](YarnServiceAPI.html) for detailed API documentations.
The API-server is stateless, which means users can simply spin up multiple instances, and have a load balancer fronting them to
support HA, distribute the load etc.
@@ -37,10 +37,10 @@ support HA, distribute the load etc.
### Service Discovery
A DNS server is implemented to enable discovering services on YARN via the standard mechanism: DNS lookup.
-The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.md). And the DNS server essentially exposes the
+The framework posts container information such as hostname and ip into the [YARN service registry](../registry/index.html). And the DNS server essentially exposes the
information in YARN service registry by translating them into DNS records such as A record and SRV record.
Clients can then discover the IPs of containers via standard DNS lookup.
The previous read mechanisms of YARN Service Registry were limited to a registry specific (java) API and a REST interface and are difficult
-to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.md)
+to wireup existing clients and services. The DNS based service discovery eliminates this gap. Please refer to this [Service Discovery doc](ServiceDiscovery.html)
for more details.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
index 72c2d3e..8e2bf9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Overview.md
@@ -46,16 +46,16 @@ The benefits of combining these workloads are two-fold:
*`This feature is in alpha state`* and so APIs, command lines are subject to change. We will continue to update the documents over time.
-[QuickStart](QuickStart.md) shows a quick tutorial that walks you through simple steps to deploy a service on YARN.
+[QuickStart](QuickStart.html) shows a quick tutorial that walks you through simple steps to deploy a service on YARN.
## How do I get my hands dirty?
-* [Concepts](Concepts.md): Describes the internals of the framework and some features in YARN core to support running services on YARN.
-* [Service REST API](YarnServiceAPI.md): The API doc for deploying/managing services on YARN.
-* [Service Discovery](ServiceDiscovery.md): Describes the service discovery mechanism on YARN.
-* [Registry DNS](RegistryDNS.md): Deep dives into the Registry DNS internals.
-* [Examples](Examples.md): List some example service definitions (`Yarnfile`).
-* [Configurations](Configurations.md): Describes how to configure the custom services on YARN.
+* [Concepts](Concepts.html): Describes the internals of the framework and some features in YARN core to support running services on YARN.
+* [Service REST API](YarnServiceAPI.html): The API doc for deploying/managing services on YARN.
+* [Service Discovery](ServiceDiscovery.html): Describes the service discovery mechanism on YARN.
+* [Registry DNS](RegistryDNS.html): Deep dives into the Registry DNS internals.
+* [Examples](Examples.html): List some example service definitions (`Yarnfile`).
+* [Configurations](Configurations.html): Describes how to configure the custom services on YARN.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index 2981a9a..f563193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -51,11 +51,11 @@ e.g. Below command launches a `sleeper` service named as `my-sleeper` on YARN.
```
yarn app -launch my-sleeper sleeper
```
-For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.md).
+For launching docker based services using YARN Service framework, please refer to [API doc](YarnServiceAPI.html).
## Manage services on YARN via CLI
Below steps walk you through deploying a services on YARN using CLI.
-Refer to [Yarn Commands](../YarnCommands.md) for the full list of commands and options.
+Refer to [Yarn Commands](../YarnCommands.html) for the full list of commands and options.
### Deploy a service
```
yarn app -launch ${SERVICE_NAME} ${PATH_TO_SERVICE_DEF_FILE}
@@ -121,7 +121,7 @@ For running inside ResourceManager, add this property to `yarn-site.xml` and res
Services can be deployed on YARN through the ResourceManager web endpoint.
-Refer to [API doc](YarnServiceAPI.md) for the detailed API specificatiosn.
+Refer to [API doc](YarnServiceAPI.html) for the detailed API specificatiosn.
### Deploy a service
@@ -194,7 +194,7 @@ A new `service` tab is added in the YARN UI2 specially to show YARN Services in
The services framework posts the data into TimelineService and the `service` UI reads data from TimelineService to render its content.
### Enable Timeline Service v2
-Please refer to [TimeLineService v2 doc](../TimelineServiceV2.md) for how to enable Timeline Service v2.
+Please refer to [TimeLineService v2 doc](../TimelineServiceV2.html) for how to enable Timeline Service v2.
### Enable new YARN UI
@@ -229,7 +229,7 @@ E.g. A typical configuration looks like below:
# Run with Docker
The above example is only for a non-docker container based service. YARN Service Framework also provides first-class support for managing docker based services.
Most of the steps for managing docker based services are the same except that in docker the `Artifact` type for a component is `DOCKER` and the Artifact `id` is the name of the docker image.
-For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.md).
+For details in how to setup docker on YARN, please check [Docker on YARN](../DockerContainers.html).
With docker support, it also opens up a set of new possibilities to implement features such as discovering service containers on YARN with DNS.
-Check [ServiceDiscovery](ServiceDiscovery.md) for more details.
+Check [ServiceDiscovery](ServiceDiscovery.html) for more details.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
index c7f19ce..2307e5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/RegistryDNS.md
@@ -15,7 +15,7 @@
# Registry DNS Server
<!-- MACRO{toc|fromDepth=0|toDepth=3} -->
-The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.md) which is backed by a zookeeper cluster.
+The document describes the internals of Registry DNS server. It is based on the [YARN service registry](../registry/index.html) which is backed by a zookeeper cluster.
## Introduction
The Registry DNS Server provides a standard DNS interface to the information posted into the YARN Registry by deployed applications. The DNS service serves the following functions:
@@ -153,7 +153,7 @@ The Registry DNS server reads its configuration properties from the yarn-site.xm
| Name | Description |
| ------------ | ------------- |
-|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.md). |
+|hadoop.registry.zk.quorum| A comma separated list of hostname:port pairs defining the zookeeper quorum for the [YARN registry](../registry/registry-configuration.html). |
| hadoop.registry.dns.enabled | The DNS functionality is enabled for the cluster. Default is false. |
| hadoop.registry.dns.domain-name | The domain name for Hadoop cluster associated records. |
| hadoop.registry.dns.bind-address | Address associated with the network interface to which the DNS listener should bind. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d919eb6e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
index 8542899..f351e23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/ServiceDiscovery.md
@@ -18,7 +18,7 @@ This document describes the mechanism of service discovery on YARN and the
steps for enabling it.
## Overview
-A [DNS server](RegistryDNS.md) is implemented to enable discovering services on YARN via
+A [DNS server](RegistryDNS.html) is implemented to enable discovering services on YARN via
the standard mechanism: DNS lookup.
The framework ApplicationMaster posts the container information such as hostname and IP address into
@@ -71,7 +71,7 @@ assigned `0` since it is the first and only instance for the `hbasemaster` compo
## Configure Registry DNS
Below is the set of configurations in `yarn-site.xml` required for enabling Registry DNS. A full list of properties can be found in the Configuration
-section of [Registry DNS](RegistryDNS.md).
+section of [Registry DNS](RegistryDNS.html).
```
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org