You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by wa...@apache.org on 2014/08/07 22:26:57 UTC
svn commit: r1616589 - in
/hadoop/common/branches/fs-encryption/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/
hadoop-yarn/hadoo...
Author: wang
Date: Thu Aug 7 20:26:52 2014
New Revision: 1616589
URL: http://svn.apache.org/r1616589
Log:
Merge from trunk to branch.
Added:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/
- copied from r1616586, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/
- copied from r1616586, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
- copied unchanged from r1616586, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
Removed:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineAuthenticator.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineAuthenticator.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
Modified:
hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/CHANGES.txt Thu Aug 7 20:26:52 2014
@@ -88,6 +88,12 @@ Release 2.6.0 - UNRELEASED
YARN-2370. Fix comment in o.a.h.y.server.resourcemanager.schedulerAppSchedulingInfo
(Wenwu Peng via junping_du)
+ YARN-2298. Move TimelineClient to yarn-common project (Zhijie Shen via
+ junping_du)
+
+ YARN-2288. Made persisted data in LevelDB timeline store be versioned. (Junping Du
+ via zjshen)
+
OPTIMIZATIONS
BUG FIXES
@@ -131,6 +137,17 @@ Release 2.6.0 - UNRELEASED
YARN-2051. Fix bug in PBimpls and add more unit tests with reflection.
(Binglin Chang via junping_du)
+ YARN-2374. Fixed TestDistributedShell#testDSShell failure due to hostname
+ dismatch. (Varun Vasudev via jianhe)
+
+ YARN-2359. Application hangs when it fails to launch AM container.
+ (Zhihai Xu via kasha)
+
+ YARN-2388. Fixed TestTimelineWebServices failure due to HADOOP-10791. (zjshen)
+
+ YARN-2008. Fixed CapacityScheduler to calculate headroom based on max available
+ capacity instead of configured max capacity. (Craig Welch via jianhe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java Thu Aug 7 20:26:52 2014
@@ -26,13 +26,13 @@ import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintWriter;
+import java.net.InetAddress;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -169,7 +169,9 @@ public class TestDistributedShell {
yarnClient.init(new Configuration(yarnCluster.getConfig()));
yarnClient.start();
String hostName = NetUtils.getHostname();
+
boolean verified = false;
+ String errorMessage = "";
while(!verified) {
List<ApplicationReport> apps = yarnClient.getApplications();
if (apps.size() == 0 ) {
@@ -177,15 +179,22 @@ public class TestDistributedShell {
continue;
}
ApplicationReport appReport = apps.get(0);
- if (appReport.getHost().startsWith(hostName)
- && appReport.getRpcPort() == -1) {
+ if(appReport.getHost().equals("N/A")) {
+ Thread.sleep(10);
+ continue;
+ }
+ errorMessage =
+ "Expected host name to start with '" + hostName + "', was '"
+ + appReport.getHost() + "'. Expected rpc port to be '-1', was '"
+ + appReport.getRpcPort() + "'.";
+ if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
verified = true;
}
if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
break;
}
}
- Assert.assertTrue(verified);
+ Assert.assertTrue(errorMessage, verified);
t.join();
LOG.info("Client run completed. Result=" + result);
Assert.assertTrue(result.get());
@@ -212,6 +221,64 @@ public class TestDistributedShell {
.toString(), ApplicationMaster.DSEntity.DS_CONTAINER.toString());
}
+ /*
+ * NetUtils.getHostname() returns a string in the form "hostname/ip".
+ * Sometimes the hostname we get is the FQDN and sometimes the short name. In
+ * addition, on machines with multiple network interfaces, it runs any one of
+ * the ips. The function below compares the returns values for
+ * NetUtils.getHostname() accounting for the conditions mentioned.
+ */
+ private boolean checkHostname(String appHostname) throws Exception {
+
+ String hostname = NetUtils.getHostname();
+ if (hostname.equals(appHostname)) {
+ return true;
+ }
+
+ Assert.assertTrue("Unknown format for hostname " + appHostname,
+ appHostname.contains("/"));
+ Assert.assertTrue("Unknown format for hostname " + hostname,
+ hostname.contains("/"));
+
+ String[] appHostnameParts = appHostname.split("/");
+ String[] hostnameParts = hostname.split("/");
+
+ return (compareFQDNs(appHostnameParts[0], hostnameParts[0]) && checkIPs(
+ hostnameParts[0], hostnameParts[1], appHostnameParts[1]));
+ }
+
+ private boolean compareFQDNs(String appHostname, String hostname)
+ throws Exception {
+ if (appHostname.equals(hostname)) {
+ return true;
+ }
+ String appFQDN = InetAddress.getByName(appHostname).getCanonicalHostName();
+ String localFQDN = InetAddress.getByName(hostname).getCanonicalHostName();
+ return appFQDN.equals(localFQDN);
+ }
+
+ private boolean checkIPs(String hostname, String localIP, String appIP)
+ throws Exception {
+
+ if (localIP.equals(appIP)) {
+ return true;
+ }
+ boolean appIPCheck = false;
+ boolean localIPCheck = false;
+ InetAddress[] addresses = InetAddress.getAllByName(hostname);
+ for (InetAddress ia : addresses) {
+ if (ia.getHostAddress().equals(appIP)) {
+ appIPCheck = true;
+ continue;
+ }
+ if (ia.getHostAddress().equals(localIP)) {
+ localIPCheck = true;
+ }
+ }
+ return (appIPCheck && localIPCheck);
+
+ }
+
@Test(timeout=90000)
public void testDSRestartWithPreviousRunningContainers() throws Exception {
String[] args = {
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml Thu Aug 7 20:26:52 2014
@@ -57,14 +57,6 @@
<groupId>log4j</groupId>
<artifactId>log4j</artifactId>
</dependency>
- <dependency>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jetty-util</artifactId>
- </dependency>
- <dependency>
- <groupId>com.sun.jersey</groupId>
- <artifactId>jersey-client</artifactId>
- </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Thu Aug 7 20:26:52 2014
@@ -67,10 +67,18 @@
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</dependency>
<dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-core-asl</artifactId>
</dependency>
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java Thu Aug 7 20:26:52 2014
@@ -42,6 +42,13 @@ public class DefaultResourceCalculator e
Resource numerator, Resource denominator) {
return ratio(numerator, denominator);
}
+
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java Thu Aug 7 20:26:52 2014
@@ -109,6 +109,14 @@ public class DominantResourceCalculator
getResourceAsValue(clusterResource, numerator, true) /
getResourceAsValue(clusterResource, denominator, true);
}
+
+ @Override
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java Thu Aug 7 20:26:52 2014
@@ -150,6 +150,15 @@ public abstract class ResourceCalculator
Resource clusterResource, Resource numerator, Resource denominator);
/**
+ * Determine if a resource is not suitable for use as a divisor
+ * (will result in divide by 0, etc)
+ *
+ * @param r resource
+ * @return true if divisor is invalid (should not be used), false else
+ */
+ public abstract boolean isInvalidDivisor(Resource r);
+
+ /**
* Ratio of resource <code>a</code> to resource <code>b</code>.
*
* @param a resource
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java Thu Aug 7 20:26:52 2014
@@ -184,6 +184,11 @@ public class Resources {
return calculator.roundDown(lhs, factor);
}
+ public static boolean isInvalidDivisor(
+ ResourceCalculator resourceCalculator, Resource divisor) {
+ return resourceCalculator.isInvalidDivisor(divisor);
+ }
+
public static float ratio(
ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) {
return resourceCalculator.ratio(lhs, rhs);
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java Thu Aug 7 20:26:52 2014
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.ti
import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.readReverseOrderedLong;
import static org.apache.hadoop.yarn.server.timeline.GenericObjectMapper.writeReverseOrderedLong;
+import static org.fusesource.leveldbjni.JniDBFactory.bytes;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -60,8 +61,12 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.VersionProto;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.server.records.impl.pb.VersionPBImpl;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DB;
+import org.iq80.leveldb.DBException;
import org.iq80.leveldb.DBIterator;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.ReadOptions;
@@ -141,6 +146,11 @@ public class LeveldbTimelineStore extend
"z".getBytes();
private static final byte[] EMPTY_BYTES = new byte[0];
+
+ private static final String TIMELINE_STORE_VERSION_KEY = "timeline-store-version";
+
+ private static final Version CURRENT_VERSION_INFO = Version
+ .newInstance(1, 0);
@Private
@VisibleForTesting
@@ -193,6 +203,7 @@ public class LeveldbTimelineStore extend
}
LOG.info("Using leveldb path " + dbPath);
db = factory.open(new File(dbPath.toString()), options);
+ checkVersion();
startTimeWriteCache =
Collections.synchronizedMap(new LRUMap(getStartTimeWriteCacheSize(
conf)));
@@ -1270,8 +1281,6 @@ public class LeveldbTimelineStore extend
DEFAULT_TIMELINE_SERVICE_LEVELDB_START_TIME_WRITE_CACHE_SIZE);
}
- // warning is suppressed to prevent eclipse from noting unclosed resource
- @SuppressWarnings("resource")
@VisibleForTesting
List<String> getEntityTypes() throws IOException {
DBIterator iterator = null;
@@ -1489,4 +1498,65 @@ public class LeveldbTimelineStore extend
readOptions.fillCache(fillCache);
return db.iterator(readOptions);
}
+
+ Version loadVersion() throws IOException {
+ byte[] data = db.get(bytes(TIMELINE_STORE_VERSION_KEY));
+ // if version is not stored previously, treat it as 1.0.
+ if (data == null || data.length == 0) {
+ return Version.newInstance(1, 0);
+ }
+ Version version =
+ new VersionPBImpl(VersionProto.parseFrom(data));
+ return version;
+ }
+
+ // Only used for test
+ @VisibleForTesting
+ void storeVersion(Version state) throws IOException {
+ dbStoreVersion(state);
+ }
+
+ private void dbStoreVersion(Version state) throws IOException {
+ String key = TIMELINE_STORE_VERSION_KEY;
+ byte[] data =
+ ((VersionPBImpl) state).getProto().toByteArray();
+ try {
+ db.put(bytes(key), data);
+ } catch (DBException e) {
+ throw new IOException(e);
+ }
+ }
+
+ Version getCurrentVersion() {
+ return CURRENT_VERSION_INFO;
+ }
+
+ /**
+ * 1) Versioning timeline store: major.minor. For e.g. 1.0, 1.1, 1.2...1.25, 2.0 etc.
+ * 2) Any incompatible change of TS-store is a major upgrade, and any
+ * compatible change of TS-store is a minor upgrade.
+ * 3) Within a minor upgrade, say 1.1 to 1.2:
+ * overwrite the version info and proceed as normal.
+ * 4) Within a major upgrade, say 1.2 to 2.0:
+ * throw exception and indicate user to use a separate upgrade tool to
+ * upgrade timeline store or remove incompatible old state.
+ */
+ private void checkVersion() throws IOException {
+ Version loadedVersion = loadVersion();
+ LOG.info("Loaded timeline store version info " + loadedVersion);
+ if (loadedVersion.equals(getCurrentVersion())) {
+ return;
+ }
+ if (loadedVersion.isCompatibleTo(getCurrentVersion())) {
+ LOG.info("Storing timeline store version info " + getCurrentVersion());
+ dbStoreVersion(CURRENT_VERSION_INFO);
+ } else {
+ String incompatibleMessage =
+ "Incompatible version for timeline store: expecting version "
+ + getCurrentVersion() + ", but loading version " + loadedVersion;
+ LOG.fatal(incompatibleMessage);
+ throw new IOException(incompatibleMessage);
+ }
+ }
+
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TestLeveldbTimelineStore.java Thu Aug 7 20:26:52 2014
@@ -36,14 +36,17 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.service.ServiceStateException;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.records.Version;
import org.apache.hadoop.yarn.server.timeline.LeveldbTimelineStore;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
import org.iq80.leveldb.DBIterator;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -52,19 +55,19 @@ import org.junit.Test;
public class TestLeveldbTimelineStore extends TimelineStoreTestUtils {
private FileContext fsContext;
private File fsPath;
+ private Configuration config = new YarnConfiguration();
@Before
public void setup() throws Exception {
fsContext = FileContext.getLocalFSFileContext();
- Configuration conf = new YarnConfiguration();
fsPath = new File("target", this.getClass().getSimpleName() +
"-tmpDir").getAbsoluteFile();
fsContext.delete(new Path(fsPath.getAbsolutePath()), true);
- conf.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH,
+ config.set(YarnConfiguration.TIMELINE_SERVICE_LEVELDB_PATH,
fsPath.getAbsolutePath());
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false);
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_TTL_ENABLE, false);
store = new LeveldbTimelineStore();
- store.init(conf);
+ store.init(config);
store.start();
loadTestData();
loadVerificationData();
@@ -263,5 +266,47 @@ public class TestLeveldbTimelineStore ex
assertEquals(1, getEntities("type_2").size());
assertEquals(2, getEntitiesWithPrimaryFilter("type_1", userFilter).size());
}
+
+ @Test
+ public void testCheckVersion() throws IOException {
+ LeveldbTimelineStore dbStore = (LeveldbTimelineStore) store;
+ // default version
+ Version defaultVersion = dbStore.getCurrentVersion();
+ Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+
+ // compatible version
+ Version compatibleVersion =
+ Version.newInstance(defaultVersion.getMajorVersion(),
+ defaultVersion.getMinorVersion() + 2);
+ dbStore.storeVersion(compatibleVersion);
+ Assert.assertEquals(compatibleVersion, dbStore.loadVersion());
+ restartTimelineStore();
+ dbStore = (LeveldbTimelineStore) store;
+ // overwrite the compatible version
+ Assert.assertEquals(defaultVersion, dbStore.loadVersion());
+
+ // incompatible version
+ Version incompatibleVersion =
+ Version.newInstance(defaultVersion.getMajorVersion() + 1,
+ defaultVersion.getMinorVersion());
+ dbStore.storeVersion(incompatibleVersion);
+ try {
+ restartTimelineStore();
+ Assert.fail("Incompatible version, should expect fail here.");
+ } catch (ServiceStateException e) {
+ Assert.assertTrue("Exception message mismatch",
+ e.getMessage().contains("Incompatible version for timeline store"));
+ }
+ }
+
+ private void restartTimelineStore() throws IOException {
+ // need to close so leveldb releases database lock
+ if (store != null) {
+ store.close();
+ }
+ store = new LeveldbTimelineStore();
+ store.init(config);
+ store.start();
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java Thu Aug 7 20:26:52 2014
@@ -33,6 +33,7 @@ import java.util.Map;
import java.util.Set;
import javax.servlet.FilterConfig;
+import javax.servlet.ServletContext;
import javax.servlet.ServletException;
import javax.ws.rs.core.MediaType;
@@ -105,6 +106,8 @@ public class TestTimelineWebServices ext
.thenReturn("simple");
when(filterConfig.getInitParameter(
PseudoAuthenticationHandler.ANONYMOUS_ALLOWED)).thenReturn("true");
+ ServletContext context = mock(ServletContext.class);
+ when(filterConfig.getServletContext()).thenReturn(context);
Enumeration<Object> names = mock(Enumeration.class);
when(names.hasMoreElements()).thenReturn(true, true, false);
when(names.nextElement()).thenReturn(
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Thu Aug 7 20:26:52 2014
@@ -217,7 +217,13 @@ public class RMAppAttemptImpl implements
RMAppAttemptEventType.KILL,
new FinalSavingTransition(new BaseFinalTransition(
RMAppAttemptState.KILLED), RMAppAttemptState.KILLED))
-
+ .addTransition(RMAppAttemptState.SCHEDULED,
+ RMAppAttemptState.FINAL_SAVING,
+ RMAppAttemptEventType.CONTAINER_FINISHED,
+ new FinalSavingTransition(
+ new AMContainerCrashedBeforeRunningTransition(),
+ RMAppAttemptState.FAILED))
+
// Transitions from ALLOCATED_SAVING State
.addTransition(RMAppAttemptState.ALLOCATED_SAVING,
RMAppAttemptState.ALLOCATED,
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Thu Aug 7 20:26:52 2014
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -24,6 +27,8 @@ import org.apache.hadoop.yarn.util.resou
class CSQueueUtils {
+ private static final Log LOG = LogFactory.getLog(CSQueueUtils.class);
+
final static float EPSILON = 0.0001f;
public static void checkMaxCapacity(String queueName,
@@ -113,4 +118,52 @@ class CSQueueUtils {
)
);
}
+
+ public static float getAbsoluteMaxAvailCapacity(
+ ResourceCalculator resourceCalculator, Resource clusterResource, CSQueue queue) {
+ CSQueue parent = queue.getParent();
+ if (parent == null) {
+ return queue.getAbsoluteMaximumCapacity();
+ }
+
+ //Get my parent's max avail, needed to determine my own
+ float parentMaxAvail = getAbsoluteMaxAvailCapacity(
+ resourceCalculator, clusterResource, parent);
+ //...and as a resource
+ Resource parentResource = Resources.multiply(clusterResource, parentMaxAvail);
+
+ //check for no resources parent before dividing, if so, max avail is none
+ if (Resources.isInvalidDivisor(resourceCalculator, parentResource)) {
+ return 0.0f;
+ }
+ //sibling used is parent used - my used...
+ float siblingUsedCapacity = Resources.ratio(
+ resourceCalculator,
+ Resources.subtract(parent.getUsedResources(), queue.getUsedResources()),
+ parentResource);
+ //my max avail is the lesser of my max capacity and what is unused from my parent
+ //by my siblings (if they are beyond their base capacity)
+ float maxAvail = Math.min(
+ queue.getMaximumCapacity(),
+ 1.0f - siblingUsedCapacity);
+ //and, mutiply by parent to get absolute (cluster relative) value
+ float absoluteMaxAvail = maxAvail * parentMaxAvail;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("qpath " + queue.getQueuePath());
+ LOG.debug("parentMaxAvail " + parentMaxAvail);
+ LOG.debug("siblingUsedCapacity " + siblingUsedCapacity);
+ LOG.debug("getAbsoluteMaximumCapacity " + queue.getAbsoluteMaximumCapacity());
+ LOG.debug("maxAvail " + maxAvail);
+ LOG.debug("absoluteMaxAvail " + absoluteMaxAvail);
+ }
+
+ if (absoluteMaxAvail < 0.0f) {
+ absoluteMaxAvail = 0.0f;
+ } else if (absoluteMaxAvail > 1.0f) {
+ absoluteMaxAvail = 1.0f;
+ }
+
+ return absoluteMaxAvail;
+ }
}
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java Thu Aug 7 20:26:52 2014
@@ -976,13 +976,18 @@ public class LeafQueue implements CSQueu
Resource userLimit = // User limit
computeUserLimit(application, clusterResource, required);
-
+
+ //Max avail capacity needs to take into account usage by ancestor-siblings
+ //which are greater than their base capacity, so we are interested in "max avail"
+ //capacity
+ float absoluteMaxAvailCapacity = CSQueueUtils.getAbsoluteMaxAvailCapacity(
+ resourceCalculator, clusterResource, this);
Resource queueMaxCap = // Queue Max-Capacity
Resources.multiplyAndNormalizeDown(
resourceCalculator,
clusterResource,
- absoluteMaxCapacity,
+ absoluteMaxAvailCapacity,
minimumAllocation);
Resource userConsumed = getUser(user).getConsumedResources();
Modified: hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java?rev=1616589&r1=1616588&r2=1616589&view=diff
==============================================================================
--- hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java (original)
+++ hadoop/common/branches/fs-encryption/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java Thu Aug 7 20:26:52 2014
@@ -89,6 +89,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
@@ -783,6 +784,32 @@ public class TestRMAppAttemptTransitions
}
@Test
+ public void testAMCrashAtScheduled() {
+ // This is to test sending CONTAINER_FINISHED event at SCHEDULED state.
+ // Verify the state transition is correct.
+ scheduleApplicationAttempt();
+ ContainerStatus cs =
+ SchedulerUtils.createAbnormalContainerStatus(
+ BuilderUtils.newContainerId(
+ applicationAttempt.getAppAttemptId(), 1),
+ SchedulerUtils.LOST_CONTAINER);
+ // send CONTAINER_FINISHED event at SCHEDULED state,
+ // The state should be FINAL_SAVING with previous state SCHEDULED
+ applicationAttempt.handle(new RMAppAttemptContainerFinishedEvent(
+ applicationAttempt.getAppAttemptId(), cs));
+ // createApplicationAttemptState will return previous state (SCHEDULED),
+ // if the current state is FINAL_SAVING.
+ assertEquals(YarnApplicationAttemptState.SCHEDULED,
+ applicationAttempt.createApplicationAttemptState());
+ // send ATTEMPT_UPDATE_SAVED event,
+ // verify the state is changed to state FAILED.
+ sendAttemptUpdateSavedEvent(applicationAttempt);
+ assertEquals(RMAppAttemptState.FAILED,
+ applicationAttempt.getAppAttemptState());
+ verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
+ }
+
+ @Test
public void testAllocatedToKilled() {
Container amContainer = allocateApplicationAttempt();
applicationAttempt.handle(