You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by vi...@apache.org on 2013/08/02 21:26:17 UTC
svn commit: r1509831 - in
/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src:
main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ test/
test/java/ test/jav...
Author: vinodkv
Date: Fri Aug 2 19:26:17 2013
New Revision: 1509831
URL: http://svn.apache.org/r1509831
Log:
YARN-956. Missed new files in the previous commit.
Added:
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/
hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
Added: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java?rev=1509831&view=auto
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java (added)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryStore.java Fri Aug 2 19:26:17 2013
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public interface ApplicationHistoryStore extends ApplicationHistoryReader,
+ ApplicationHistoryWriter {
+}
Added: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java?rev=1509831&view=auto
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java (added)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/MemoryApplicationHistoryStore.java Fri Aug 2 19:26:17 2013
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+
+public class MemoryApplicationHistoryStore implements ApplicationHistoryStore {
+
+ private static MemoryApplicationHistoryStore memStore = null;
+
+ private ConcurrentHashMap<ApplicationId, ApplicationHistoryData> applicationData =
+ new ConcurrentHashMap<ApplicationId, ApplicationHistoryData>();
+ private ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>> applicationAttemptData =
+ new ConcurrentHashMap<ApplicationId, ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>>();
+ private ConcurrentHashMap<ContainerId, ContainerHistoryData> containerData =
+ new ConcurrentHashMap<ContainerId, ContainerHistoryData>();
+
+ private MemoryApplicationHistoryStore() {
+ }
+
+ public static MemoryApplicationHistoryStore getMemoryStore() {
+ if (memStore == null) {
+ memStore = new MemoryApplicationHistoryStore();
+ }
+ return memStore;
+ }
+
+ @Override
+ public Map<ApplicationId, ApplicationHistoryData> getAllApplications() {
+ Map<ApplicationId, ApplicationHistoryData> listApps =
+ new HashMap<ApplicationId, ApplicationHistoryData>();
+ for (ApplicationId appId : applicationData.keySet()) {
+ listApps.put(appId, applicationData.get(appId));
+ }
+ return listApps;
+ }
+
+ @Override
+ public ApplicationHistoryData getApplication(ApplicationId appId) {
+ return applicationData.get(appId);
+ }
+
+ @Override
+ public Map<ApplicationAttemptId, ApplicationAttemptHistoryData> getApplicationAttempts(
+ ApplicationId appId) {
+ Map<ApplicationAttemptId, ApplicationAttemptHistoryData> listAttempts =
+ null;
+ ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+ applicationAttemptData.get(appId);
+ if (appAttempts != null) {
+ listAttempts =
+ new HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
+ for (ApplicationAttemptId attemptId : appAttempts.keySet()) {
+ listAttempts.put(attemptId, appAttempts.get(attemptId));
+ }
+ }
+ return listAttempts;
+ }
+
+ @Override
+ public ApplicationAttemptHistoryData getApplicationAttempt(
+ ApplicationAttemptId appAttemptId) {
+ ApplicationAttemptHistoryData appAttemptHistoryData = null;
+ ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+ applicationAttemptData.get(appAttemptId.getApplicationId());
+ if (appAttempts != null) {
+ appAttemptHistoryData = appAttempts.get(appAttemptId);
+ }
+ return appAttemptHistoryData;
+ }
+
+ @Override
+ public ContainerHistoryData getAMContainer(ApplicationAttemptId appAttemptId) {
+ ContainerHistoryData Container = null;
+ ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+ applicationAttemptData.get(appAttemptId.getApplicationId());
+ if (appAttempts != null) {
+ containerData.get(appAttempts.get(appAttemptId).getMasterContainerId());
+ }
+ return Container;
+ }
+
+ @Override
+ public ContainerHistoryData getContainer(ContainerId containerId) {
+ return containerData.get(containerId);
+ }
+
+ @Override
+ public void writeApplication(ApplicationHistoryData app) throws Throwable {
+ if (app != null) {
+ ApplicationHistoryData oldData =
+ applicationData.putIfAbsent(app.getApplicationId(), app);
+ if (oldData != null) {
+ throw new IOException("This application "
+ + app.getApplicationId().toString() + " is already present.");
+ }
+ }
+ }
+
+ @Override
+ public void writeApplicationAttempt(ApplicationAttemptHistoryData appAttempt)
+ throws Throwable {
+ if (appAttempt != null) {
+ if (applicationAttemptData.containsKey(appAttempt
+ .getApplicationAttemptId().getApplicationId())) {
+ ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
+ applicationAttemptData.get(appAttempt.getApplicationAttemptId()
+ .getApplicationId());
+ ApplicationAttemptHistoryData oldAppAttempt =
+ appAttemptmap.putIfAbsent(appAttempt.getApplicationAttemptId(),
+ appAttempt);
+ if (oldAppAttempt != null) {
+ throw new IOException("This application attempt "
+ + appAttempt.getApplicationAttemptId().toString()
+ + " already present.");
+ }
+ } else {
+ ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttemptmap =
+ new ConcurrentHashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>();
+ appAttemptmap.put(appAttempt.getApplicationAttemptId(), appAttempt);
+ applicationAttemptData.putIfAbsent(appAttempt.getApplicationAttemptId()
+ .getApplicationId(), appAttemptmap);
+ }
+ }
+ }
+
+ @Override
+ public void writeContainer(ContainerHistoryData container) throws Throwable {
+ if (container != null) {
+ ContainerHistoryData oldContainer =
+ containerData.putIfAbsent(container.getContainerId(), container);
+ if (oldContainer != null) {
+ throw new IOException("This container "
+ + container.getContainerId().toString() + " is already present.");
+ }
+ }
+ }
+}
Added: hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1509831&view=auto
==============================================================================
--- hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java (added)
+++ hadoop/common/branches/YARN-321/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java Fri Aug 2 19:26:17 2013
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.applicationhistoryservice;
+
+import java.util.HashMap;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationAttemptHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ApplicationHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.ContainerHistoryData;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationAttemptHistoryDataPBImpl;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ApplicationHistoryDataPBImpl;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.records.impl.pb.ContainerHistoryDataPBImpl;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMemoryApplicationHistoryStore {
+ MemoryApplicationHistoryStore memstore = null;
+
+ @Before
+ public void setup() throws Throwable {
+ memstore = MemoryApplicationHistoryStore.getMemoryStore();
+ writeHistoryApplication();
+ writeHistoryApplicationAttempt();
+ writeContainer();
+ }
+
+ public void writeHistoryApplication() throws Throwable {
+ ApplicationHistoryData appData = new ApplicationHistoryDataPBImpl();
+ appData.setApplicationId(ApplicationId.newInstance(1234, 1));
+ memstore.writeApplication(appData);
+ }
+
+ public void writeHistoryApplicationAttempt() throws Throwable {
+ ApplicationAttemptHistoryData appAttemptHistoryData =
+ new ApplicationAttemptHistoryDataPBImpl();
+ appAttemptHistoryData.setApplicationAttemptId(ApplicationAttemptId
+ .newInstance(ApplicationId.newInstance(1234, 1), 1));
+ memstore.writeApplicationAttempt(appAttemptHistoryData);
+ }
+
+ public void writeContainer() throws Throwable {
+ ContainerHistoryData container = new ContainerHistoryDataPBImpl();
+ container.setContainerId(ContainerId.newInstance(ApplicationAttemptId
+ .newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
+ memstore.writeContainer(container);
+ }
+
+ @After
+ public void tearDown() {
+ }
+
+ @Test
+ public void testReadApplication() {
+ HashMap<ApplicationId, ApplicationHistoryData> map =
+ (HashMap<ApplicationId, ApplicationHistoryData>) memstore
+ .getAllApplications();
+ Assert.assertEquals(1, map.size());
+ ApplicationHistoryData appData = null;
+ for (ApplicationId appId : map.keySet()) {
+ appData = map.get(appId);
+ Assert.assertEquals("application_1234_0001", appData.getApplicationId()
+ .toString());
+ }
+ HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData> appAttempts =
+ (HashMap<ApplicationAttemptId, ApplicationAttemptHistoryData>) memstore
+ .getApplicationAttempts(appData.getApplicationId());
+ Assert.assertEquals(1, appAttempts.size());
+ ApplicationAttemptHistoryData appAttempt = null;
+ for (ApplicationAttemptId appAttemptId : appAttempts.keySet()) {
+ appAttempt = appAttempts.get(appAttemptId);
+ Assert.assertEquals("appattempt_1234_0001_000001", appAttempt
+ .getApplicationAttemptId().toString());
+ }
+ ContainerHistoryData amContainer =
+ memstore.getContainer(ContainerId.newInstance(ApplicationAttemptId
+ .newInstance(ApplicationId.newInstance(1234, 1), 1), 1));
+ Assert.assertEquals("container_1234_0001_01_000001", amContainer
+ .getContainerId().toString());
+ }
+}