You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2014/01/28 20:50:29 UTC
svn commit: r1562182 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn:
hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn-api/src/main/proto/ hadoop-yarn-api/src/main/proto/server/
hadoop-yarn-common/src/...
Author: vinodkv
Date: Tue Jan 28 19:50:28 2014
New Revision: 1562182
URL: http://svn.apache.org/r1562182
Log:
YARN-934. Updated patch for HistoryWriter to split the write operations into a startOp and a finishOp. Contributed by Zhijie Shen.
YARN-947. Updated patch to have separate data object for startOp and finishOp. Contributed by Zhijie Shen.
YARN-956. Updated patch for In-memory Storage after the above two changes.
svn merge --ignore-ancestry -c 1556726 ../YARN-321
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/YarnApplicationAttemptState.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptFinishData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptStartData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationFinishData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationStartData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerFinishData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerStartData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptFinishDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptStartDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationFinishDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationStartDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerFinishDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerStartDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
- copied unchanged from r1556726, hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStoreTestUtils.java
Removed:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationAttemptHistoryDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ApplicationHistoryDataPBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/impl/pb/ContainerHistoryDataPBImpl.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/server/application_history_server.proto Tue Jan 28 19:50:28 2014
@@ -35,6 +35,25 @@ message ApplicationHistoryDataProto {
optional int64 finish_time = 8;
optional string diagnostics_info = 9;
optional FinalApplicationStatusProto final_application_status = 10;
+ optional YarnApplicationStateProto yarn_application_state = 11;
+}
+
+message ApplicationStartDataProto {
+ optional ApplicationIdProto application_id = 1;
+ optional string application_name = 2;
+ optional string application_type = 3;
+ optional string user = 4;
+ optional string queue = 5;
+ optional int64 submit_time = 6;
+ optional int64 start_time = 7;
+}
+
+message ApplicationFinishDataProto {
+ optional ApplicationIdProto application_id = 1;
+ optional int64 finish_time = 2;
+ optional string diagnostics_info = 3;
+ optional FinalApplicationStatusProto final_application_status = 4;
+ optional YarnApplicationStateProto yarn_application_state = 5;
}
message ApplicationAttemptHistoryDataProto {
@@ -45,6 +64,22 @@ message ApplicationAttemptHistoryDataPro
optional string diagnostics_info = 5;
optional FinalApplicationStatusProto final_application_status = 6;
optional ContainerIdProto master_container_id = 7;
+ optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 8;
+}
+
+message ApplicationAttemptStartDataProto {
+ optional ApplicationAttemptIdProto application_attempt_id = 1;
+ optional string host = 2;
+ optional int32 rpc_port = 3;
+ optional ContainerIdProto master_container_id = 4;
+}
+
+message ApplicationAttemptFinishDataProto {
+ optional ApplicationAttemptIdProto application_attempt_id = 1;
+ optional string tracking_url = 2;
+ optional string diagnostics_info = 3;
+ optional FinalApplicationStatusProto final_application_status = 4;
+ optional YarnApplicationAttemptStateProto yarn_application_attempt_state = 5;
}
message ContainerHistoryDataProto {
@@ -56,5 +91,23 @@ message ContainerHistoryDataProto {
optional int64 finish_time = 6;
optional string diagnostics_info = 7;
optional string log_url = 8;
- optional ContainerStateProto final_container_status = 9;
+ optional int32 container_exit_status = 9;
+ optional ContainerStateProto container_state = 10;
+}
+
+message ContainerStartDataProto {
+ optional ContainerIdProto container_id = 1;
+ optional ResourceProto allocated_resource = 2;
+ optional NodeIdProto assigned_node_id = 3;
+ optional PriorityProto priority = 4;
+ optional int64 start_time = 5;
+}
+
+message ContainerFinishDataProto {
+ optional ContainerIdProto container_id = 1;
+ optional int64 finish_time = 2;
+ optional string diagnostics_info = 3;
+ optional string log_url = 4;
+ optional int32 container_exit_status = 5;
+ optional ContainerStateProto container_state = 6;
}
\ No newline at end of file
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Tue Jan 28 19:50:28 2014
@@ -98,6 +98,21 @@ enum YarnApplicationStateProto {
KILLED = 8;
}
+enum YarnApplicationAttemptStateProto {
+ APP_ATTEMPT_NEW = 1;
+ APP_ATTEMPT_SUBMITTED = 2;
+ APP_ATTEMPT_SCHEDULED = 3;
+ APP_ATTEMPT_ALLOCATED_SAVING = 4;
+ APP_ATTEMPT_ALLOCATED = 5;
+ APP_ATTEMPT_LAUNCHED = 6;
+ APP_ATTEMPT_FAILED = 7;
+ APP_ATTEMPT_RUNNING = 8;
+ APP_ATTEMPT_FINAL_SAVING = 9;
+ APP_ATTEMPT_FINISHING = 10;
+ APP_ATTEMPT_FINISHED = 11;
+ APP_ATTEMPT_KILLED = 12;
+}
+
enum FinalApplicationStatusProto {
APP_UNDEFINED = 0;
APP_SUCCEEDED = 1;
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java Tue Jan 28 19:50:28 2014
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.proto.YarnProtos.AMCommandProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationAccessTypeProto;
@@ -45,6 +46,7 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnProtos.NodeStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueACLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.QueueStateProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationAttemptStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnApplicationStateProto;
import com.google.protobuf.ByteString;
@@ -97,6 +99,21 @@ public class ProtoUtils {
}
/*
+ * YarnApplicationAttemptState
+ */
+ private static String YARN_APPLICATION_ATTEMPT_STATE_PREFIX = "APP_ATTEMPT_";
+ public static YarnApplicationAttemptStateProto convertToProtoFormat(
+ YarnApplicationAttemptState e) {
+ return YarnApplicationAttemptStateProto
+ .valueOf(YARN_APPLICATION_ATTEMPT_STATE_PREFIX + e.name());
+ }
+ public static YarnApplicationAttemptState convertFromProtoFormat(
+ YarnApplicationAttemptStateProto e) {
+ return YarnApplicationAttemptState.valueOf(e.name().replace(
+ YARN_APPLICATION_ATTEMPT_STATE_PREFIX, ""));
+ }
+
+ /*
* ApplicationResourceUsageReport
*/
public static ApplicationResourceUsageReportProto convertToProtoFormat(ApplicationResourceUsageReport e) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java Tue Jan 28 19:50:28 2014
@@ -20,9 +20,18 @@ package org.apache.hadoop.yarn.server.ap
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
+/**
+ * This class is the abstract of the storage of the application history data. It
+ * is a {@link Service}, such that the implementation of this class can make use
+ * of the service life cycle to initialize and cleanup the storage. Users can
+ * access the storage via {@link ApplicationHistoryReader} and
+ * {@link ApplicationHistoryWriter} interfaces.
+ *
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public interface ApplicationHistoryStore extends ApplicationHistoryReader,
- ApplicationHistoryWriter {
+public interface ApplicationHistoryStore extends Service,
+ ApplicationHistoryReader, ApplicationHistoryWriter {
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryWriter.java Tue Jan 28 19:50:28 2014
@@ -18,11 +18,16 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
+import java.io.IOException;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
/**
* It is the interface of writing the application history, exposing the methods
@@ -34,25 +39,72 @@ import org.apache.hadoop.yarn.server.app
public interface ApplicationHistoryWriter {
/**
- * This method persists an {@link ApplicationHistoryData} object.
- * @param app the {@link ApplicationHistoryData} object
- * @throws Throwable
+ * This method writes the information of <code>RMApp</code> that is available
+ * when it starts.
+ *
+ * @param appStart
+ * the record of the information of <code>RMApp</code> that is
+ * available when it starts
+ * @throws IOException
+ */
+ void applicationStarted(ApplicationStartData appStart) throws IOException;
+
+ /**
+ * This method writes the information of <code>RMApp</code> that is available
+ * when it finishes.
+ *
+ * @param appFinish
+ * the record of the information of <code>RMApp</code> that is
+ * available when it finishes
+ * @throws IOException
+ */
+ void applicationFinished(ApplicationFinishData appFinish) throws IOException;
+
+ /**
+ * This method writes the information of <code>RMAppAttempt</code> that is
+ * available when it starts.
+ *
+ * @param appAttemptStart
+ * the record of the information of <code>RMAppAttempt</code> that is
+ * available when it starts
+ * @throws IOException
+ */
+ void applicationAttemptStarted(
+ ApplicationAttemptStartData appAttemptStart) throws IOException;
+
+ /**
+ * This method writes the information of <code>RMAppAttempt</code> that is
+ * available when it finishes.
+ *
+ * @param appAttemptFinish
+ * the record of the information of <code>RMAppAttempt</code> that is
+ * available when it finishes
+ * @throws IOException
*/
- void writeApplication(ApplicationHistoryData app) throws Throwable;
+ void applicationAttemptFinished(
+ ApplicationAttemptFinishData appAttemptFinish) throws IOException;
/**
- * This method persists an {@link ApplicationAttemptHistoryData} object.
- * @param appAttempt the {@link ApplicationAttemptHistoryData} object
- * @throws Throwable
+ * This method writes the information of <code>RMContainer</code> that is
+ * available when it starts.
+ *
+ * @param containerStart
+ * the record of the information of <code>RMContainer</code> that is
+ * available when it starts
+ * @throws IOException
*/
- void writeApplicationAttempt(
- ApplicationAttemptHistoryData appAttempt) throws Throwable;
+ void containerStarted(ContainerStartData containerStart) throws IOException;
/**
- * This method persists a {@link ContainerHistoryData} object.
- * @param container the {@link ContainerHistoryData} object
- * @throws Throwable
+ * This method writes the information of <code>RMContainer</code> that is
+ * available when it finishes.
+ *
+ * @param containerFinish
+ * the record of the information of <code>RMContainer</code> that is
+ * available when it finishes
+ * @throws IOException
*/
- void writeContainer(ContainerHistoryData container) throws Throwable;
+ void containerFinished(ContainerFinishData containerFinish)
+ throws IOException;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java Tue Jan 28 19:50:28 2014
@@ -19,46 +19,55 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
+import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationStartData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerFinishData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerStartData;
-public class MemoryApplicationHistoryStore implements ApplicationHistoryStore {
+/**
+ * In-memory implementation of {@link ApplicationHistoryStore}.
+ * This implementation is for test purpose only. If users improperly instantiate
+ * it, they may encounter reading and writing history data in different memory
+ * store.
+ *
+ */
+@Private
+@Unstable
+public class MemoryApplicationHistoryStore extends AbstractService
+ implements ApplicationHistoryStore {
- private static MemoryApplicationHistoryStore memStore = null;
-
- private ConcurrentHashMap<ApplicationId, ApplicationHistoryData> applicationData =
+ private final ConcurrentMap<ApplicationId, ApplicationHistoryData> applicationData =
new ConcurrentHashMap<ApplicationId, ApplicationHistoryData>();
- private ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
- new ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
- private ConcurrentHashMap<ContainerId, ContainerHistoryData> containerData =
- new ConcurrentHashMap<ContainerId, ContainerHistoryData>();
+ private final ConcurrentMap<ApplicationId, ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
+ new ConcurrentHashMap<ApplicationId, ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
+ private final ConcurrentMap<ApplicationAttemptId, ConcurrentMap<ContainerId, ContainerHistoryData>> containerData =
+ new ConcurrentHashMap<ApplicationAttemptId, ConcurrentMap<ContainerId, ContainerHistoryData>>();
- private MemoryApplicationHistoryStore() {
- }
-
- public static MemoryApplicationHistoryStore getMemoryStore() {
- if (memStore == null) {
- memStore = new MemoryApplicationHistoryStore();
- }
- return memStore;
+ public MemoryApplicationHistoryStore() {
+ super(MemoryApplicationHistoryStore.class.getName());
}
@Override
public Map<ApplicationId, ApplicationHistoryData> getAllApplications() {
- Map<ApplicationId, ApplicationHistoryData> listApps =
- new HashMap<ApplicationId, ApplicationHistoryData>();
- for (ApplicationId appId : applicationData.keySet()) {
- listApps.put(appId, applicationData.get(appId));
- }
- return listApps;
+ return new HashMap<ApplicationId, ApplicationHistoryData>(
+ applicationData);
}
@Override
@@ -67,111 +76,209 @@ public class MemoryApplicationHistorySto
}
@Override
- public Map<ApplicationAttemptId, ApplicationAttemptHistoryData> getApplicationAttempts(
- ApplicationId appId) {
- Map<ApplicationAttemptId, ApplicationAttemptHistoryData> listAttempts =
- null;
- ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+ public Map<ApplicationAttemptId, ApplicationAttemptHistoryData>
+ getApplicationAttempts(
+ ApplicationId appId) {
+ ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
applicationAttemptData.get(appId);
- if (appAttempts != null) {
- listAttempts =
- new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
- for (ApplicationAttemptId attemptId : appAttempts.keySet()) {
- listAttempts.put(attemptId, appAttempts.get(attemptId));
- }
+ if (subMap == null) {
+ return Collections.<ApplicationAttemptId, ApplicationAttemptHistoryData>emptyMap();
+ } else {
+ return new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>(subMap);
}
- return listAttempts;
}
@Override
public ApplicationAttemptHistoryData getApplicationAttempt(
ApplicationAttemptId appAttemptId) {
- ApplicationAttemptHistoryData appAttemptHistoryData = null;
- ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+ ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
applicationAttemptData.get(appAttemptId.getApplicationId());
- if (appAttempts != null) {
- appAttemptHistoryData = appAttempts.get(appAttemptId);
+ if (subMap == null) {
+ return null;
+ } else {
+ return subMap.get(appAttemptId);
}
- return appAttemptHistoryData;
}
@Override
- public ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) {
- ContainerHistoryData Container = null;
- ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
- applicationAttemptData.get(appAttemptId.getApplicationId());
- if (appAttempts != null) {
- containerData.get(appAttempts.get(appAttemptId).getMasterContainerId());
+ public ContainerHistoryData getAMContainer(
+ ApplicationAttemptId appAttemptId) {
+ ApplicationAttemptHistoryData appAttempt =
+ getApplicationAttempt(appAttemptId);
+ if (appAttempt == null || appAttempt.getMasterContainerId() == null) {
+ return null;
+ } else {
+ return getContainer(appAttempt.getMasterContainerId());
}
- return Container;
}
@Override
public ContainerHistoryData getContainer(ContainerId containerId) {
- return containerData.get(containerId);
+ Map<ContainerId, ContainerHistoryData> subMap =
+ containerData.get(containerId.getApplicationAttemptId());
+ if (subMap == null) {
+ return null;
+ } else {
+ return subMap.get(containerId);
+ }
}
@Override
- public void writeApplication(ApplicationHistoryData app) throws Throwable {
- if (app != null) {
- ApplicationHistoryData oldData =
- applicationData.putIfAbsent(app.getApplicationId(), app);
- if (oldData != null) {
- throw new IOException("This application "
- + app.getApplicationId().toString() + " is already present.");
- }
+ public Map<ContainerId, ContainerHistoryData> getContainers(
+ ApplicationAttemptId appAttemptId) throws IOException {
+ ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
+ containerData.get(appAttemptId);
+ if (subMap == null) {
+ return Collections.<ContainerId, ContainerHistoryData>emptyMap();
+ } else {
+ return new HashMap<ContainerId, ContainerHistoryData>(subMap);
}
}
@Override
- public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt)
- throws Throwable {
- if (appAttempt != null) {
- if (applicationAttemptData.containsKey(appAttempt
- .getApplicationAttemptId().getApplicationId())) {
- ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
- applicationAttemptData.get(appAttempt.getApplicationAttemptId()
- .getApplicationId());
- ApplicationAttemptHistoryData oldAppAttempt =
- appAttemptmap.putIfAbsent(appAttempt.getApplicationAttemptId(),
- appAttempt);
- if (oldAppAttempt != null) {
- throw new IOException("This application attempt "
- + appAttempt.getApplicationAttemptId().toString()
- + " already present.");
- }
- } else {
- ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
- new ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
- appAttemptmap.put(appAttempt.getApplicationAttemptId(), appAttempt);
- applicationAttemptData.putIfAbsent(appAttempt.getApplicationAttemptId()
- .getApplicationId(), appAttemptmap);
- }
+ public void applicationStarted(ApplicationStartData appStart)
+ throws IOException {
+ ApplicationHistoryData oldData =
+ applicationData.putIfAbsent(appStart.getApplicationId(),
+ ApplicationHistoryData.newInstance(
+ appStart.getApplicationId(),
+ appStart.getApplicationName(),
+ appStart.getApplicationType(),
+ appStart.getQueue(),
+ appStart.getUser(),
+ appStart.getSubmitTime(),
+ appStart.getStartTime(),
+ Long.MAX_VALUE, null, null, null));
+ if (oldData != null) {
+ throw new IOException("The start information of application "
+ + appStart.getApplicationId() + " is already stored.");
}
}
@Override
- public void writeContainer(ContainerHistoryData container) throws Throwable {
- if (container != null) {
- ContainerHistoryData oldContainer =
- containerData.putIfAbsent(container.getContainerId(), container);
- if (oldContainer != null) {
- throw new IOException("This container "
- + container.getContainerId().toString() + " is already present.");
- }
+ public void applicationFinished(ApplicationFinishData appFinish)
+ throws IOException {
+ ApplicationHistoryData data =
+ applicationData.get(appFinish.getApplicationId());
+ if (data == null) {
+ throw new IOException("The finish information of application "
+ + appFinish.getApplicationId() + " is stored before the start"
+ + " information.");
+ }
+ // Make the assumption that YarnApplicationState should not be null if
+ // the finish information is already recorded
+ if (data.getYarnApplicationState() != null) {
+ throw new IOException("The finish information of application "
+ + appFinish.getApplicationId() + " is already stored.");
}
+ data.setFinishTime(appFinish.getFinishTime());
+ data.setDiagnosticsInfo(appFinish.getDiagnosticsInfo());
+ data.setFinalApplicationStatus(appFinish.getFinalApplicationStatus());
+ data.setYarnApplicationState(appFinish.getYarnApplicationState());
}
@Override
- public Map<ContainerId, ContainerHistoryData> getContainers(
- ApplicationAttemptId appAttemptId) throws IOException {
- HashMap<ContainerId, ContainerHistoryData> containers =
- new HashMap<ContainerId, ContainerHistoryData>();
- for (ContainerId container : containerData.keySet()) {
- if (container.getApplicationAttemptId().equals(appAttemptId)) {
- containers.put(container, containerData.get(container));
- }
+ public void applicationAttemptStarted(
+ ApplicationAttemptStartData appAttemptStart) throws IOException {
+ ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
+ getSubMap(appAttemptStart.getApplicationAttemptId().getApplicationId());
+ ApplicationAttemptHistoryData oldData = subMap.putIfAbsent(
+ appAttemptStart.getApplicationAttemptId(),
+ ApplicationAttemptHistoryData.newInstance(
+ appAttemptStart.getApplicationAttemptId(),
+ appAttemptStart.getHost(),
+ appAttemptStart.getRPCPort(),
+ appAttemptStart.getMasterContainerId(),
+ null, null, null, null));
+ if (oldData != null) {
+ throw new IOException("The start information of application attempt "
+ + appAttemptStart.getApplicationAttemptId()
+ + " is already stored.");
}
- return containers;
}
+
+ @Override
+ public void applicationAttemptFinished(
+ ApplicationAttemptFinishData appAttemptFinish) throws IOException {
+ ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData> subMap =
+ getSubMap(appAttemptFinish.getApplicationAttemptId().getApplicationId());
+ ApplicationAttemptHistoryData data =
+ subMap.get(appAttemptFinish.getApplicationAttemptId());
+ if (data == null) {
+ throw new IOException("The finish information of application attempt "
+ + appAttemptFinish.getApplicationAttemptId() + " is stored before"
+ + " the start information.");
+ }
+ // Make the assumption that YarnApplicationAttemptState should not be null
+ // if the finish information is already recorded
+ if (data.getYarnApplicationAttemptState() != null) {
+ throw new IOException("The finish information of application attempt "
+ + appAttemptFinish.getApplicationAttemptId()
+ + " is already stored.");
+ }
+ data.setTrackingURL(appAttemptFinish.getTrackingURL());
+ data.setDiagnosticsInfo(appAttemptFinish.getDiagnosticsInfo());
+ data.setFinalApplicationStatus(appAttemptFinish.getFinalApplicationStatus());
+ data.setYarnApplicationAttemptState(appAttemptFinish.getYarnApplicationAttemptState());
+ }
+
+ private ConcurrentMap<ApplicationAttemptId, ApplicationAttemptHistoryData>
+ getSubMap(ApplicationId appId) {
+ applicationAttemptData.putIfAbsent(appId,
+ new ConcurrentHashMap<ApplicationAttemptId,
+ ApplicationAttemptHistoryData>());
+ return applicationAttemptData.get(appId);
+ }
+
+ @Override
+ public void containerStarted(ContainerStartData containerStart)
+ throws IOException {
+ ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
+ getSubMap(containerStart.getContainerId().getApplicationAttemptId());
+ ContainerHistoryData oldData = subMap.putIfAbsent(
+ containerStart.getContainerId(),
+ ContainerHistoryData.newInstance(
+ containerStart.getContainerId(),
+ containerStart.getAllocatedResource(),
+ containerStart.getAssignedNode(),
+ containerStart.getPriority(),
+ containerStart.getStartTime(),
+ Long.MAX_VALUE, null, null, Integer.MAX_VALUE, null));
+ if (oldData != null) {
+ throw new IOException("The start information of container "
+ + containerStart.getContainerId() + " is already stored.");
+ }
+ }
+
+ @Override
+ public void containerFinished(ContainerFinishData containerFinish)
+ throws IOException {
+ ConcurrentMap<ContainerId, ContainerHistoryData> subMap =
+ getSubMap(containerFinish.getContainerId().getApplicationAttemptId());
+ ContainerHistoryData data = subMap.get(containerFinish.getContainerId());
+ if (data == null) {
+ throw new IOException("The finish information of container "
+ + containerFinish.getContainerId() + " is stored before"
+ + " the start information.");
+ }
+ // Make the assumption that ContainerState should not be null if
+ // the finish information is already recorded
+ if (data.getContainerState() != null) {
+ throw new IOException("The finish information of container "
+ + containerFinish.getContainerId() + " is already stored.");
+ }
+ data.setFinishTime(containerFinish.getFinishTime());
+ data.setDiagnosticsInfo(containerFinish.getDiagnosticsInfo());
+ data.setLogURL(containerFinish.getLogURL());
+ data.setContainerExitStatus(containerFinish.getContainerExitStatus());
+ data.setContainerState(containerFinish.getContainerState());
+ }
+
+ private ConcurrentMap<ContainerId, ContainerHistoryData> getSubMap(
+ ApplicationAttemptId appAttemptId) {
+ containerData.putIfAbsent(appAttemptId,
+ new ConcurrentHashMap<ContainerId, ContainerHistoryData>());
+ return containerData.get(appAttemptId);
+ }
+
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationAttemptHistoryData.java Tue Jan 28 19:50:28 2014
@@ -23,69 +23,150 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
/**
- * The class contains all the fields that need to be stored persistently for
+ * The class contains all the fields that are stored persistently for
* <code>RMAppAttempt</code>.
*/
@Public
@Unstable
-public interface ApplicationAttemptHistoryData {
+public class ApplicationAttemptHistoryData {
+
+ private ApplicationAttemptId applicationAttemptId;
+
+ private String host;
+
+ private int rpcPort;
+
+ private String trackingURL;
+
+ private String diagnosticsInfo;
+
+ private FinalApplicationStatus finalApplicationStatus;
+
+ private ContainerId masterContainerId;
+
+ private YarnApplicationAttemptState yarnApplicationAttemptState;
+
+ @Public
+ @Unstable
+ public static ApplicationAttemptHistoryData newInstance(
+ ApplicationAttemptId appAttemptId, String host, int rpcPort,
+ ContainerId masterContainerId, String diagnosticsInfo,
+ String trackingURL,
+ FinalApplicationStatus finalApplicationStatus,
+ YarnApplicationAttemptState yarnApplicationAttemptState) {
+ ApplicationAttemptHistoryData appAttemptHD =
+ new ApplicationAttemptHistoryData();
+ appAttemptHD.setApplicationAttemptId(appAttemptId);
+ appAttemptHD.setHost(host);
+ appAttemptHD.setRPCPort(rpcPort);
+ appAttemptHD.setMasterContainerId(masterContainerId);
+ appAttemptHD.setDiagnosticsInfo(diagnosticsInfo);
+ appAttemptHD.setTrackingURL(trackingURL);
+ appAttemptHD.setFinalApplicationStatus(finalApplicationStatus);
+ appAttemptHD.setYarnApplicationAttemptState(yarnApplicationAttemptState);
+ return appAttemptHD;
+ }
+
+ @Public
+ @Unstable
+ public ApplicationAttemptId getApplicationAttemptId() {
+ return applicationAttemptId;
+ }
+
+ @Public
+ @Unstable
+ public void setApplicationAttemptId(
+ ApplicationAttemptId applicationAttemptId) {
+ this.applicationAttemptId = applicationAttemptId;
+ }
@Public
@Unstable
- ApplicationAttemptId getApplicationAttemptId();
+ public String getHost() {
+ return host;
+ }
@Public
@Unstable
- void setApplicationAttemptId(ApplicationAttemptId applicationAttemptId);
+ public void setHost(String host) {
+ this.host = host;
+ }
@Public
@Unstable
- String getHost();
+ public int getRPCPort() {
+ return rpcPort;
+ }
@Public
@Unstable
- void setHost(String host);
+ public void setRPCPort(int rpcPort) {
+ this.rpcPort = rpcPort;
+ }
@Public
@Unstable
- int getRPCPort();
+ public String getTrackingURL() {
+ return trackingURL;
+ }
@Public
@Unstable
- void setRPCPort(int rpcPort);
+ public void setTrackingURL(String trackingURL) {
+ this.trackingURL = trackingURL;
+ }
@Public
@Unstable
- String getTrackingURL();
+ public String getDiagnosticsInfo() {
+ return diagnosticsInfo;
+ }
@Public
@Unstable
- void setTrackingURL(String trackingURL);
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ this.diagnosticsInfo = diagnosticsInfo;
+ }
@Public
@Unstable
- String getDiagnosticsInfo();
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return finalApplicationStatus;
+ }
@Public
@Unstable
- void setDiagnosticsInfo(String diagnosticsInfo);
+ public void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus) {
+ this.finalApplicationStatus = finalApplicationStatus;
+ }
@Public
@Unstable
- FinalApplicationStatus getFinalApplicationStatus();
+ public ContainerId getMasterContainerId() {
+ return masterContainerId;
+ }
@Public
@Unstable
- void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus);
+ public void setMasterContainerId(ContainerId masterContainerId) {
+ this.masterContainerId = masterContainerId;
+ }
@Public
@Unstable
- ContainerId getMasterContainerId();
+ public YarnApplicationAttemptState getYarnApplicationAttemptState() {
+ return yarnApplicationAttemptState;
+ }
@Public
@Unstable
- void setMasterContainerId(ContainerId masterContainerId);
+ public void setYarnApplicationAttemptState(
+ YarnApplicationAttemptState yarnApplicationAttemptState) {
+ this.yarnApplicationAttemptState = yarnApplicationAttemptState;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ApplicationHistoryData.java Tue Jan 28 19:50:28 2014
@@ -22,93 +22,192 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
/**
- * The class contains all the fields that need to be stored persistently for
+ * The class contains all the fields that are stored persistently for
* <code>RMApp</code>.
*/
@Public
@Unstable
-public interface ApplicationHistoryData {
+public class ApplicationHistoryData {
+
+ private ApplicationId applicationId;
+
+ private String applicationName;
+
+ private String applicationType;
+
+ private String user;
+
+ private String queue;
+
+ private long submitTime;
+
+ private long startTime;
+
+ private long finishTime;
+
+ private String diagnosticsInfo;
+
+ private FinalApplicationStatus finalApplicationStatus;
+
+ private YarnApplicationState yarnApplicationState;
+
+ @Public
+ @Unstable
+ public static ApplicationHistoryData newInstance(ApplicationId applicationId,
+ String applicationName, String applicationType, String queue,
+ String user, long submitTime, long startTime, long finishTime,
+ String diagnosticsInfo, FinalApplicationStatus finalApplicationStatus,
+ YarnApplicationState yarnApplicationState) {
+ ApplicationHistoryData appHD = new ApplicationHistoryData();
+ appHD.setApplicationId(applicationId);
+ appHD.setApplicationName(applicationName);
+ appHD.setApplicationType(applicationType);
+ appHD.setQueue(queue);
+ appHD.setUser(user);
+ appHD.setSubmitTime(submitTime);
+ appHD.setStartTime(startTime);
+ appHD.setFinishTime(finishTime);
+ appHD.setDiagnosticsInfo(diagnosticsInfo);
+ appHD.setFinalApplicationStatus(finalApplicationStatus);
+ appHD.setYarnApplicationState(yarnApplicationState);
+ return appHD;
+ }
+
+ @Public
+ @Unstable
+ public ApplicationId getApplicationId() {
+ return applicationId;
+ }
+
+ @Public
+ @Unstable
+ public void setApplicationId(ApplicationId applicationId) {
+ this.applicationId = applicationId;
+ }
@Public
@Unstable
- ApplicationId getApplicationId();
+ public String getApplicationName() {
+ return applicationName;
+ }
@Public
@Unstable
- void setApplicationId(ApplicationId applicationId);
+ public void setApplicationName(String applicationName) {
+ this.applicationName = applicationName;
+ }
@Public
@Unstable
- String getApplicationName();
+ public String getApplicationType() {
+ return applicationType;
+ }
@Public
@Unstable
- void setApplicationName(String applicationName);
+ public void setApplicationType(String applicationType) {
+ this.applicationType = applicationType;
+ }
@Public
@Unstable
- String getApplicationType();
+ public String getUser() {
+ return user;
+ }
@Public
@Unstable
- void setApplicationType(String applicationType);
+ public void setUser(String user) {
+ this.user = user;
+ }
@Public
@Unstable
- String getUser();
+ public String getQueue() {
+ return queue;
+ }
@Public
@Unstable
- void setUser(String user);
+ public void setQueue(String queue) {
+ this.queue = queue;
+ }
@Public
@Unstable
- String getQueue();
+ public long getSubmitTime() {
+ return submitTime;
+ }
@Public
@Unstable
- void setQueue(String queue);
+ public void setSubmitTime(long submitTime) {
+ this.submitTime = submitTime;
+ }
@Public
@Unstable
- long getSubmitTime();
+ public long getStartTime() {
+ return startTime;
+ }
@Public
@Unstable
- void setSubmitTime(long submitTime);
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
@Public
@Unstable
- long getStartTime();
+ public long getFinishTime() {
+ return finishTime;
+ }
@Public
@Unstable
- void setStartTime(long startTime);
+ public void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
@Public
@Unstable
- long getFinishTime();
+ public String getDiagnosticsInfo() {
+ return diagnosticsInfo;
+ }
@Public
@Unstable
- void setFinishTime(long finishTime);
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ this.diagnosticsInfo = diagnosticsInfo;
+ }
@Public
@Unstable
- String getDiagnosticsInfo();
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return finalApplicationStatus;
+ }
@Public
@Unstable
- void setDiagnosticsInfo(String diagnosticInfo);
+ public void setFinalApplicationStatus(
+ FinalApplicationStatus finalApplicationStatus) {
+ this.finalApplicationStatus = finalApplicationStatus;
+ }
@Public
@Unstable
- FinalApplicationStatus getFinalApplicationStatus();
+ public YarnApplicationState getYarnApplicationState() {
+ return this.yarnApplicationState;
+ }
@Public
@Unstable
- void setFinalApplicationStatus(FinalApplicationStatus finalApplicationStatus);
+ public void setYarnApplicationState(
+ YarnApplicationState yarnApplicationState) {
+ this.yarnApplicationState = yarnApplicationState;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/records/ContainerHistoryData.java Tue Jan 28 19:50:28 2014
@@ -27,83 +27,171 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
/**
- * The class contains all the fields that need to be stored persistently for
+ * The class contains all the fields that are stored persistently for
* <code>RMContainer</code>.
*/
@Public
@Unstable
-public interface ContainerHistoryData {
+public class ContainerHistoryData {
+
+ private ContainerId containerId;
+
+ private Resource allocatedResource;
+
+ private NodeId assignedNode;
+
+ private Priority priority;
+
+ private long startTime;
+
+ private long finishTime;
+
+ private String diagnosticsInfo;
+
+ private String logURL;
+
+ private int containerExitStatus;
+
+ private ContainerState containerState;
+
+ @Public
+ @Unstable
+ public static ContainerHistoryData newInstance(ContainerId containerId,
+ Resource allocatedResource, NodeId assignedNode, Priority priority,
+ long startTime, long finishTime, String diagnosticsInfo, String logURL,
+ int containerExitCode, ContainerState containerState) {
+ ContainerHistoryData containerHD = new ContainerHistoryData();
+ containerHD.setContainerId(containerId);
+ containerHD.setAllocatedResource(allocatedResource);
+ containerHD.setAssignedNode(assignedNode);
+ containerHD.setPriority(priority);
+ containerHD.setStartTime(startTime);
+ containerHD.setFinishTime(finishTime);
+ containerHD.setDiagnosticsInfo(diagnosticsInfo);
+ containerHD.setLogURL(logURL);
+ containerHD.setContainerExitStatus(containerExitCode);
+ containerHD.setContainerState(containerState);
+ return containerHD;
+ }
+
+ @Public
+ @Unstable
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ @Public
+ @Unstable
+ public void setContainerId(ContainerId containerId) {
+ this.containerId = containerId;
+ }
@Public
@Unstable
- ContainerId getContainerId();
+ public Resource getAllocatedResource() {
+ return allocatedResource;
+ }
@Public
@Unstable
- void setContainerId(ContainerId containerId);
+ public void setAllocatedResource(Resource resource) {
+ this.allocatedResource = resource;
+ }
@Public
@Unstable
- Resource getAllocatedResource();
+ public NodeId getAssignedNode() {
+ return assignedNode;
+ }
@Public
@Unstable
- void setAllocatedResource(Resource resource);
+ public void setAssignedNode(NodeId nodeId) {
+ this.assignedNode = nodeId;
+ }
@Public
@Unstable
- NodeId getAssignedNode();
+ public Priority getPriority() {
+ return priority;
+ }
@Public
@Unstable
- void setAssignedNode(NodeId nodeId);
+ public void setPriority(Priority priority) {
+ this.priority = priority;
+ }
@Public
@Unstable
- Priority getPriority();
+ public long getStartTime() {
+ return startTime;
+ }
@Public
@Unstable
- void setPriority(Priority priority);
+ public void setStartTime(long startTime) {
+ this.startTime = startTime;
+ }
@Public
@Unstable
- long getStartTime();
+ public long getFinishTime() {
+ return finishTime;
+ }
@Public
@Unstable
- void setStartTime(long startTime);
+ public void setFinishTime(long finishTime) {
+ this.finishTime = finishTime;
+ }
@Public
@Unstable
- long getFinishTime();
+ public String getDiagnosticsInfo() {
+ return diagnosticsInfo;
+ }
@Public
@Unstable
- void setFinishTime(long finishTime);
+ public void setDiagnosticsInfo(String diagnosticsInfo) {
+ this.diagnosticsInfo = diagnosticsInfo;
+ }
@Public
@Unstable
- String getDiagnosticsInfo();
+ public String getLogURL() {
+ return logURL;
+ }
@Public
@Unstable
- void setDiagnosticsInfo(String diagnosticInfo);
+ public void setLogURL(String logURL) {
+ this.logURL = logURL;
+ }
@Public
@Unstable
- String getLogURL();
+ public int getContainerExitStatus() {
+ return containerExitStatus;
+ }
@Public
@Unstable
- void setLogURL(String logURL);
+ public void setContainerExitStatus(int containerExitStatus) {
+ this.containerExitStatus = containerExitStatus;
+ }
@Public
@Unstable
- ContainerState getFinalContainerStatus();
+ public ContainerState getContainerState() {
+ return containerState;
+ }
@Public
@Unstable
- void setFinalContainerStatus(ContainerState finalContainerState);
+ public void setContainerState(ContainerState containerState) {
+ this.containerState = containerState;
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java Tue Jan 28 19:50:28 2014
@@ -19,102 +19,189 @@
package org.apache.hadoop.yarn.server.applicationhistoryservice;
import java.io.IOException;
-import java.util.HashMap;
import junit.framework.Assert;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationAttemptHistoryDataPBImpl;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ContainerHistoryDataPBImpl;
-import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-public class TestMemoryApplicationHistoryStore {
- MemoryApplicationHistoryStore memstore = null;
+public class TestMemoryApplicationHistoryStore extends
+ ApplicationHistoryStoreTestUtils {
@Before
- public void setup() throws Throwable {
- memstore = MemoryApplicationHistoryStore.getMemoryStore();
- writeHistoryApplication();
- writeHistoryApplicationAttempt();
- writeContainer();
+ public void setup() {
+ store = new MemoryApplicationHistoryStore();
}
- public void writeHistoryApplication() throws Throwable {
- ApplicationHistoryData appData = new ApplicationHistoryDataPBImpl();
- appData.setApplicationId(ApplicationId.newInstance(1234, 1));
- memstore.writeApplication(appData);
- }
-
- public void writeHistoryApplicationAttempt() throws Throwable {
- ApplicationAttemptHistoryData appAttemptHistoryData =
- new ApplicationAttemptHistoryDataPBImpl();
- appAttemptHistoryData.setApplicationAttemptId(ApplicationAttemptId
- .newInstance(ApplicationId.newInstance(1234, 1), 1));
- memstore.writeApplicationAttempt(appAttemptHistoryData);
+ @Test
+ public void testReadWriteApplicationHistory() throws Exception {
+ // Out of order
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ try {
+ writeApplicationFinishData(appId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "is stored before the start information"));
+ }
+ // Normal
+ int numApps = 5;
+ for (int i = 1; i <= numApps; ++i) {
+ appId = ApplicationId.newInstance(0, i);
+ writeApplicationStartData(appId);
+ writeApplicationFinishData(appId);
+ }
+ Assert.assertEquals(numApps, store.getAllApplications().size());
+ for (int i = 1; i <= numApps; ++i) {
+ appId = ApplicationId.newInstance(0, i);
+ ApplicationHistoryData data = store.getApplication(appId);
+ Assert.assertNotNull(data);
+ Assert.assertEquals(appId.toString(), data.getApplicationName());
+ Assert.assertEquals(appId.toString(), data.getDiagnosticsInfo());
+ }
+ // Write again
+ appId = ApplicationId.newInstance(0, 1);
+ try {
+ writeApplicationStartData(appId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is already stored"));
+ }
+ try {
+ writeApplicationFinishData(appId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is already stored"));
+ }
}
- public void writeContainer() throws Throwable {
- ContainerHistoryData container = new ContainerHistoryDataPBImpl();
- container.setContainerId(ContainerId.newInstance(ApplicationAttemptId
- .newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
- memstore.writeContainer(container);
+ @Test
+ public void testReadWriteApplicationAttemptHistory() throws Exception {
+ // Out of order
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ try {
+ writeApplicationAttemptFinishData(appAttemptId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "is stored before the start information"));
+ }
+ // Normal
+ int numAppAttempts = 5;
+ writeApplicationStartData(appId);
+ for (int i = 1; i <= numAppAttempts; ++i) {
+ appAttemptId =
+ ApplicationAttemptId.newInstance(appId, i);
+ writeApplicationAttemptStartData(appAttemptId);
+ writeApplicationAttemptFinishData(appAttemptId);
+ }
+ Assert.assertEquals(
+ numAppAttempts, store.getApplicationAttempts(appId).size());
+ for (int i = 1; i <= numAppAttempts; ++i) {
+ appAttemptId =
+ ApplicationAttemptId.newInstance(appId, i);
+ ApplicationAttemptHistoryData data =
+ store.getApplicationAttempt(appAttemptId);
+ Assert.assertNotNull(data);
+ Assert.assertEquals(appAttemptId.toString(), data.getHost());
+ Assert.assertEquals(appAttemptId.toString(), data.getDiagnosticsInfo());
+ }
+ writeApplicationFinishData(appId);
+ // Write again
+ appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ try {
+ writeApplicationAttemptStartData(appAttemptId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is already stored"));
+ }
+ try {
+ writeApplicationAttemptFinishData(appAttemptId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is already stored"));
+ }
}
- public ContainerHistoryData writeContainer(ApplicationAttemptId appAttemptId,
- int containerId) throws Throwable {
- ContainerHistoryData container = new ContainerHistoryDataPBImpl();
- container
- .setContainerId(ContainerId.newInstance(appAttemptId, containerId));
- memstore.writeContainer(container);
- return container;
- }
-
- @After
- public void tearDown() {
+ @Test
+ public void testReadWriteContainerHistory() throws Exception {
+ // Out of order
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
+ try {
+ writeContainerFinishData(containerId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "is stored before the start information"));
+ }
+ // Normal
+ writeApplicationAttemptStartData(appAttemptId);
+ int numContainers = 5;
+ for (int i = 1; i <= numContainers; ++i) {
+ containerId = ContainerId.newInstance(appAttemptId, i);
+ writeContainerStartData(containerId);
+ writeContainerFinishData(containerId);
+ }
+ Assert.assertEquals(
+ numContainers, store.getContainers(appAttemptId).size());
+ for (int i = 1; i <= numContainers; ++i) {
+ containerId = ContainerId.newInstance(appAttemptId, i);
+ ContainerHistoryData data = store.getContainer(containerId);
+ Assert.assertNotNull(data);
+ Assert.assertEquals(Priority.newInstance(containerId.getId()),
+ data.getPriority());
+ Assert.assertEquals(containerId.toString(), data.getDiagnosticsInfo());
+ }
+ ContainerHistoryData masterContainer = store.getAMContainer(appAttemptId);
+ Assert.assertNotNull(masterContainer);
+ Assert.assertEquals(ContainerId.newInstance(appAttemptId, 1),
+ masterContainer.getContainerId());
+ writeApplicationAttemptFinishData(appAttemptId);
+ // Write again
+ containerId = ContainerId.newInstance(appAttemptId, 1);
+ try {
+ writeContainerStartData(containerId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is already stored"));
+ }
+ try {
+ writeContainerFinishData(containerId);
+ Assert.fail();
+ } catch (IOException e) {
+ Assert.assertTrue(e.getMessage().contains("is already stored"));
+ }
}
@Test
- public void testReadApplication() throws Throwable {
- HashMap<ApplicationId, ApplicationHistoryData> map =
- (HashMap<ApplicationId, ApplicationHistoryData>) memstore
- .getAllApplications();
- Assert.assertEquals(1, map.size());
- ApplicationHistoryData appData = null;
- for (ApplicationId appId : map.keySet()) {
- appData = map.get(appId);
- Assert.assertEquals("application_1234_0001", appData.getApplicationId()
- .toString());
- }
- HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
- (HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>) memstore
- .getApplicationAttempts(appData.getApplicationId());
- Assert.assertEquals(1, appAttempts.size());
- ApplicationAttemptHistoryData appAttempt = null;
- for (ApplicationAttemptId appAttemptId : appAttempts.keySet()) {
- appAttempt = appAttempts.get(appAttemptId);
- Assert.assertEquals("appattempt_1234_0001_000001", appAttempt
- .getApplicationAttemptId().toString());
- }
- ContainerHistoryData amContainer =
- memstore.getContainer(ContainerId.newInstance(ApplicationAttemptId
- .newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
- Assert.assertEquals("container_1234_0001_01_000001", amContainer
- .getContainerId().toString());
- ContainerHistoryData container2 =
- writeContainer(appAttempt.getApplicationAttemptId(), 2);
- HashMap<ContainerId, ContainerHistoryData> containers =
- (HashMap<ContainerId, ContainerHistoryData>) memstore
- .getContainers(appAttempt.getApplicationAttemptId());
- Assert.assertEquals(2, containers.size());
- Assert.assertEquals("container_1234_0001_01_000002", containers.get(
- container2.getContainerId()).getContainerId().toString());
+ public void testMassiveWriteContainerHistory() throws IOException {
+ long mb = 1024 * 1024;
+ Runtime runtime = Runtime.getRuntime();
+ long usedMemoryBefore = (runtime.totalMemory() - runtime.freeMemory()) / mb;
+ int numContainers = 100000;
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ for (int i = 1; i <= numContainers; ++i) {
+ ContainerId containerId = ContainerId.newInstance(appAttemptId, i);
+ writeContainerStartData(containerId);
+ writeContainerFinishData(containerId);
+ }
+ long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
+ Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 100);
}
+
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java Tue Jan 28 19:50:28 2014
@@ -33,9 +33,14 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -43,6 +48,7 @@ import org.apache.hadoop.yarn.server.res
* Utility methods to aid serving RM data through the REST and RPC APIs
*/
public class RMServerUtils {
+
public static List<RMNode> queryRMNodes(RMContext context,
EnumSet<NodeState> acceptedStates) {
// nodes contains nodes that are NEW, RUNNING OR UNHEALTHY
@@ -56,7 +62,7 @@ public class RMServerUtils {
}
}
}
-
+
// inactiveNodes contains nodes that are DECOMMISSIONED, LOST, OR REBOOTED
if (acceptedStates.contains(NodeState.DECOMMISSIONED) ||
acceptedStates.contains(NodeState.LOST) ||
@@ -69,7 +75,7 @@ public class RMServerUtils {
}
return results;
}
-
+
/**
* Utility method to validate a list resource requests, by insuring that the
* requested memory/vcore is non-negative and not greater than max
@@ -85,8 +91,9 @@ public class RMServerUtils {
* @throw <code>InvalidResourceBlacklistRequestException </code> if the
* resource is not able to be added to the blacklist.
*/
- public static void validateBlacklistRequest(ResourceBlacklistRequest blacklistRequest)
- throws InvalidResourceBlacklistRequestException {
+ public static void validateBlacklistRequest(
+ ResourceBlacklistRequest blacklistRequest)
+ throws InvalidResourceBlacklistRequestException {
if (blacklistRequest != null) {
List<String> plus = blacklistRequest.getBlacklistAdditions();
if (plus != null && plus.contains(ResourceRequest.ANY)) {
@@ -100,10 +107,12 @@ public class RMServerUtils {
* It will validate to make sure all the containers belong to correct
* application attempt id. If not then it will throw
* {@link InvalidContainerReleaseException}
- * @param containerReleaseList containers to be released as requested by
- * application master.
- * @param appAttemptId Application attempt Id
- * @throws InvalidContainerReleaseException
+ *
+ * @param containerReleaseList
+ * containers to be released as requested by application master.
+ * @param appAttemptId
+ * Application attempt Id
+ * @throws InvalidContainerReleaseException
*/
public static void
validateContainerReleaseRequest(List<ContainerId> containerReleaseList,
@@ -111,9 +120,11 @@ public class RMServerUtils {
throws InvalidContainerReleaseException {
for (ContainerId cId : containerReleaseList) {
if (!appAttemptId.equals(cId.getApplicationAttemptId())) {
- throw new InvalidContainerReleaseException("Cannot release container : "
- + cId.toString() + " not belonging to this application attempt : "
- + appAttemptId);
+ throw new InvalidContainerReleaseException(
+ "Cannot release container : "
+ + cId.toString()
+ + " not belonging to this application attempt : "
+ + appAttemptId);
}
}
}
@@ -157,4 +168,63 @@ public class RMServerUtils {
}
return user;
}
+
+ public static YarnApplicationState createApplicationState(
+ RMAppState rmAppState) {
+ switch (rmAppState) {
+ case NEW:
+ return YarnApplicationState.NEW;
+ case NEW_SAVING:
+ return YarnApplicationState.NEW_SAVING;
+ case SUBMITTED:
+ return YarnApplicationState.SUBMITTED;
+ case ACCEPTED:
+ return YarnApplicationState.ACCEPTED;
+ case RUNNING:
+ return YarnApplicationState.RUNNING;
+ case FINISHING:
+ case FINISHED:
+ return YarnApplicationState.FINISHED;
+ case KILLED:
+ return YarnApplicationState.KILLED;
+ case FAILED:
+ return YarnApplicationState.FAILED;
+ default:
+ throw new YarnRuntimeException("Unknown state passed!");
+ }
+ }
+
+ public static YarnApplicationAttemptState createApplicationAttemptState(
+ RMAppAttemptState rmAppAttemptState) {
+ switch (rmAppAttemptState) {
+ case NEW:
+ return YarnApplicationAttemptState.NEW;
+ case SUBMITTED:
+ return YarnApplicationAttemptState.SUBMITTED;
+ case SCHEDULED:
+ return YarnApplicationAttemptState.SCHEDULED;
+ case ALLOCATED:
+ return YarnApplicationAttemptState.ALLOCATED;
+ case LAUNCHED:
+ return YarnApplicationAttemptState.LAUNCHED;
+ case ALLOCATED_SAVING:
+ case LAUNCHED_UNMANAGED_SAVING:
+ return YarnApplicationAttemptState.ALLOCATED_SAVING;
+ case RUNNING:
+ return YarnApplicationAttemptState.RUNNING;
+ case FINISHING:
+ return YarnApplicationAttemptState.FINISHING;
+ case FINAL_SAVING:
+ return YarnApplicationAttemptState.FINAL_SAVING;
+ case FINISHED:
+ return YarnApplicationAttemptState.FINISHED;
+ case KILLED:
+ return YarnApplicationAttemptState.KILLED;
+ case FAILED:
+ return YarnApplicationAttemptState.FAILED;
+ default:
+ throw new YarnRuntimeException("Unknown state passed!");
+ }
+ }
+
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1562182&r1=1562181&r2=1562182&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Tue Jan 28 19:50:28 2014
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.ApplicationState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
@@ -1069,27 +1070,7 @@ public class RMAppImpl implements RMApp,
if (rmAppState.equals(RMAppState.KILLING)) {
rmAppState = stateBeforeKilling;
}
- switch (rmAppState) {
- case NEW:
- return YarnApplicationState.NEW;
- case NEW_SAVING:
- return YarnApplicationState.NEW_SAVING;
- case SUBMITTED:
- return YarnApplicationState.SUBMITTED;
- case ACCEPTED:
- return YarnApplicationState.ACCEPTED;
- case RUNNING:
- return YarnApplicationState.RUNNING;
- case FINISHING:
- case FINISHED:
- return YarnApplicationState.FINISHED;
- case KILLED:
- return YarnApplicationState.KILLED;
- case FAILED:
- return YarnApplicationState.FAILED;
- default:
- throw new YarnRuntimeException("Unknown state passed!");
- }
+ return RMServerUtils.createApplicationState(rmAppState);
}
public static boolean isAppInFinalState(RMApp rmApp) {