You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2015/08/25 20:08:30 UTC
[01/50] [abbrv] hadoop git commit: YARN-3040. Make putEntities
operation be aware of the app's context. Contributed by Zhijie Shen
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 3c36922d7 -> be95107aa (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index 5adae71..0f51656 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -138,7 +138,7 @@ public class TimelineCollectorWebService {
LOG.error("Application not found");
throw new NotFoundException(); // different exception?
}
- collector.postEntities(entities, callerUgi);
+ collector.putEntities(entities, callerUgi);
return Response.ok().build();
} catch (Exception e) {
LOG.error("Error putting entities", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index f5603f6..41b6ac9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -52,7 +52,9 @@ public class FileSystemTimelineWriterImpl extends AbstractService
/** default value for storage location on local disk */
public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
- = "/tmp/timeline_service_data/";
+ = "/tmp/timeline_service_data";
+
+ private static final String ENTITIES_DIR = "entities";
/** Default extension for output files */
public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
@@ -61,38 +63,25 @@ public class FileSystemTimelineWriterImpl extends AbstractService
super((FileSystemTimelineWriterImpl.class.getName()));
}
- /**
- * Stores the entire information in {@link TimelineEntity} to the
- * timeline store. Any errors occurring for individual write request objects
- * will be reported in the response.
- *
- * @param data
- * a {@link TimelineEntity} object
- * @return {@link TimelineWriteResponse} object.
- * @throws IOException
- */
@Override
- public TimelineWriteResponse write(TimelineEntities entities)
- throws IOException {
+ public TimelineWriteResponse write(String clusterId, String userId,
+ String flowId, String flowRunId, String appId,
+ TimelineEntities entities) throws IOException {
TimelineWriteResponse response = new TimelineWriteResponse();
for (TimelineEntity entity : entities.getEntities()) {
- write(entity, response);
+ write(clusterId, userId, flowId, flowRunId, appId, entity, response);
}
return response;
}
- private void write(TimelineEntity entity,
+ private void write(String clusterId, String userId,
+ String flowId, String flowRunId, String appId, TimelineEntity entity,
TimelineWriteResponse response) throws IOException {
PrintWriter out = null;
try {
- File outputDir = new File(outputRoot + entity.getType());
- String fileName = outputDir + "/" + entity.getId()
- + TIMELINE_SERVICE_STORAGE_EXTENSION;
- if (!outputDir.exists()) {
- if (!outputDir.mkdirs()) {
- throw new IOException("Could not create directories for " + fileName);
- }
- }
+ String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowId,
+ flowRunId, appId, entity.getType());
+ String fileName = dir + entity.getId() + TIMELINE_SERVICE_STORAGE_EXTENSION;
out = new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)));
out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
out.write("\n");
@@ -112,20 +101,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
}
}
- /**
- * Aggregates the entity information to the timeline store based on which
- * track this entity is to be rolled up to The tracks along which aggregations
- * are to be done are given by {@link TimelineAggregationTrack}
- *
- * Any errors occurring for individual write request objects will be reported
- * in the response.
- *
- * @param data
- * a {@link TimelineEntity} object
- * a {@link TimelineAggregationTrack} enum value
- * @return a {@link TimelineWriteResponse} object.
- * @throws IOException
- */
+ @Override
public TimelineWriteResponse aggregate(TimelineEntity data,
TimelineAggregationTrack track) throws IOException {
return null;
@@ -141,4 +117,23 @@ public class FileSystemTimelineWriterImpl extends AbstractService
outputRoot = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT);
}
+
+ @Override
+ public void serviceStart() throws Exception {
+ mkdirs(outputRoot, ENTITIES_DIR);
+ }
+
+ private static String mkdirs(String... dirStrs) throws IOException {
+ StringBuilder path = new StringBuilder();
+ for (String dirStr : dirStrs) {
+ path.append(dirStr).append('/');
+ File dir = new File(path.toString());
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new IOException("Could not create directories for " + dir);
+ }
+ }
+ }
+ return path.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 71ad7ab..492e3a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -39,12 +39,19 @@ public interface TimelineWriter extends Service {
* timeline store. Any errors occurring for individual write request objects
* will be reported in the response.
*
+ * @param clusterId context cluster ID
+ * @param userId context user ID
+ * @param flowId context flow ID
+ * @param flowRunId context flow run ID
+ * @param appId context app ID
* @param data
* a {@link TimelineEntities} object.
* @return a {@link TimelineWriteResponse} object.
* @throws IOException
*/
- TimelineWriteResponse write(TimelineEntities data) throws IOException;
+ TimelineWriteResponse write(String clusterId, String userId,
+ String flowId, String flowRunId, String appId,
+ TimelineEntities data) throws IOException;
/**
* Aggregates the entity information to the timeline store based on which
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index 3b20352..1de8d6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
@@ -29,16 +30,25 @@ import static org.mockito.Mockito.when;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
+import org.junit.After;
import org.junit.Test;
+import java.io.IOException;
+
public class TestPerNodeTimelineCollectorsAuxService {
private ApplicationAttemptId appAttemptId;
+ private PerNodeTimelineCollectorsAuxService auxService;
public TestPerNodeTimelineCollectorsAuxService() {
ApplicationId appId =
@@ -46,10 +56,16 @@ public class TestPerNodeTimelineCollectorsAuxService {
appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
}
+ @After
+ public void tearDown() throws Shell.ExitCodeException {
+ if (auxService != null) {
+ auxService.stop();
+ }
+ }
+
@Test
public void testAddApplication() throws Exception {
- PerNodeTimelineCollectorsAuxService auxService =
- createCollectorAndAddApplication();
+ auxService = createCollectorAndAddApplication();
// auxService should have a single app
assertTrue(auxService.hasApplication(
appAttemptId.getApplicationId().toString()));
@@ -58,7 +74,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
@Test
public void testAddApplicationNonAMContainer() throws Exception {
- PerNodeTimelineCollectorsAuxService auxService = createCollector();
+ auxService = createCollector();
ContainerId containerId = getContainerId(2L); // not an AM
ContainerInitializationContext context =
@@ -72,8 +88,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
@Test
public void testRemoveApplication() throws Exception {
- PerNodeTimelineCollectorsAuxService auxService =
- createCollectorAndAddApplication();
+ auxService = createCollectorAndAddApplication();
// auxService should have a single app
String appIdStr = appAttemptId.getApplicationId().toString();
assertTrue(auxService.hasApplication(appIdStr));
@@ -90,8 +105,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
@Test
public void testRemoveApplicationNonAMContainer() throws Exception {
- PerNodeTimelineCollectorsAuxService auxService =
- createCollectorAndAddApplication();
+ auxService = createCollectorAndAddApplication();
// auxService should have a single app
String appIdStr = appAttemptId.getApplicationId().toString();
assertTrue(auxService.hasApplication(appIdStr));
@@ -109,7 +123,6 @@ public class TestPerNodeTimelineCollectorsAuxService {
@Test(timeout = 60000)
public void testLaunch() throws Exception {
ExitUtil.disableSystemExit();
- PerNodeTimelineCollectorsAuxService auxService = null;
try {
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
@@ -118,10 +131,6 @@ public class TestPerNodeTimelineCollectorsAuxService {
assertEquals(0, e.status);
ExitUtil.resetFirstExitException();
fail();
- } finally {
- if (auxService != null) {
- auxService.stop();
- }
}
}
@@ -141,6 +150,8 @@ public class TestPerNodeTimelineCollectorsAuxService {
TimelineCollectorManager collectorManager = createCollectorManager();
PerNodeTimelineCollectorsAuxService auxService =
spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
+ auxService.init(new YarnConfiguration());
+ auxService.start();
return auxService;
}
@@ -150,6 +161,14 @@ public class TestPerNodeTimelineCollectorsAuxService {
doReturn(new Configuration()).when(collectorManager).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
+ GetTimelineCollectorContextResponse response =
+ GetTimelineCollectorContextResponse.newInstance(null, null, null);
+ try {
+ when(nmCollectorService.getTimelineCollectorContext(any(
+ GetTimelineCollectorContextRequest.class))).thenReturn(response);
+ } catch (YarnException | IOException e) {
+ fail();
+ }
doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
return collectorManager;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
index 541665b..38227ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -20,10 +20,14 @@ package org.apache.hadoop.yarn.server.timelineservice.collector;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.Callable;
@@ -33,15 +37,34 @@ import java.util.concurrent.Future;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TestTimelineCollectorManager {
+ private TimelineCollectorManager collectorManager;
+
+ @Before
+ public void setup() throws Exception {
+ collectorManager = createCollectorManager();
+ collectorManager.init(new YarnConfiguration());
+ collectorManager.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (collectorManager != null) {
+ collectorManager.stop();
+ }
+ }
@Test(timeout=60000)
public void testMultithreadedAdd() throws Exception {
- final TimelineCollectorManager collectorManager = createCollectorManager();
-
final int NUM_APPS = 5;
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
for (int i = 0; i < NUM_APPS; i++) {
@@ -49,7 +72,7 @@ public class TestTimelineCollectorManager {
Callable<Boolean> task = new Callable<Boolean>() {
public Boolean call() {
AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId.toString());
+ new AppLevelTimelineCollector(appId);
return (collectorManager.putIfAbsent(appId, collector) == collector);
}
};
@@ -73,8 +96,6 @@ public class TestTimelineCollectorManager {
@Test
public void testMultithreadedAddAndRemove() throws Exception {
- final TimelineCollectorManager collectorManager = createCollectorManager();
-
final int NUM_APPS = 5;
List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
for (int i = 0; i < NUM_APPS; i++) {
@@ -82,7 +103,7 @@ public class TestTimelineCollectorManager {
Callable<Boolean> task = new Callable<Boolean>() {
public Boolean call() {
AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId.toString());
+ new AppLevelTimelineCollector(appId);
boolean successPut =
(collectorManager.putIfAbsent(appId, collector) == collector);
return successPut && collectorManager.remove(appId.toString());
@@ -112,6 +133,14 @@ public class TestTimelineCollectorManager {
doReturn(new Configuration()).when(collectorManager).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
+ GetTimelineCollectorContextResponse response =
+ GetTimelineCollectorContextResponse.newInstance(null, null, null);
+ try {
+ when(nmCollectorService.getTimelineCollectorContext(any(
+ GetTimelineCollectorContextRequest.class))).thenReturn(response);
+ } catch (YarnException | IOException e) {
+ fail();
+ }
doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
return collectorManager;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index 7f919f0..407b5f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -28,9 +28,9 @@ import java.nio.file.Paths;
import java.util.List;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
@@ -52,13 +52,16 @@ public class TestFileSystemTimelineWriterImpl {
entity.setModifiedTime(1425016502000L);
te.addEntity(entity);
- try (FileSystemTimelineWriterImpl fsi =
- new FileSystemTimelineWriterImpl()) {
- fsi.serviceInit(new Configuration());
- fsi.write(te);
+ FileSystemTimelineWriterImpl fsi = null;
+ try {
+ fsi = new FileSystemTimelineWriterImpl();
+ fsi.init(new YarnConfiguration());
+ fsi.start();
+ fsi.write("cluster_id", "user_id", "flow_id", "flow_run_id", "app_id", te);
- String fileName = fsi.getOutputRoot() + "/" + type + "/" + id
- + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ String fileName = fsi.getOutputRoot() +
+ "/entities/cluster_id/user_id/flow_id/flow_run_id/app_id/" + type +
+ "/" + id + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
Path path = Paths.get(fileName);
File f = new File(fileName);
assertTrue(f.exists() && !f.isDirectory());
@@ -73,6 +76,11 @@ public class TestFileSystemTimelineWriterImpl {
File outputDir = new File(fsi.getOutputRoot());
FileUtils.deleteDirectory(outputDir);
assertTrue(!(f.exists()));
+ } finally {
+ if (fsi != null) {
+ fsi.stop();
+ FileUtils.deleteDirectory(new File(fsi.getOutputRoot()));
+ }
}
}
}
[10/50] [abbrv] hadoop git commit: YARN-3431. Sub resources of
timeline entity needs to be passed to a separate endpoint. Contributed By
Zhijie Shen.
Posted by sj...@apache.org.
YARN-3431. Sub resources of timeline entity needs to be passed to a separate endpoint. Contributed By Zhijie Shen.
(cherry picked from commit fa5cc75245a6dba549620a8b26c7b4a8aed9838e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8fd1aa15
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8fd1aa15
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8fd1aa15
Branch: refs/heads/YARN-2928
Commit: 8fd1aa15a9d11f5cd586ee98d5fc65b623e48f5c
Parents: e9e8d85
Author: Junping Du <ju...@apache.org>
Authored: Mon Apr 27 11:28:32 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../ApplicationAttemptEntity.java | 13 +-
.../timelineservice/ApplicationEntity.java | 22 +-
.../records/timelineservice/ClusterEntity.java | 12 +-
.../timelineservice/ContainerEntity.java | 13 +-
.../api/records/timelineservice/FlowEntity.java | 80 +++--
.../HierarchicalTimelineEntity.java | 124 +++----
.../records/timelineservice/QueueEntity.java | 36 +++
.../records/timelineservice/TimelineEntity.java | 322 +++++++++++++++----
.../records/timelineservice/TimelineQueue.java | 35 --
.../records/timelineservice/TimelineUser.java | 35 --
.../api/records/timelineservice/UserEntity.java | 36 +++
.../TestTimelineServiceRecords.java | 91 ++++--
.../TestTimelineServiceClientIntegration.java | 44 ++-
.../collector/TimelineCollectorWebService.java | 65 +++-
15 files changed, 654 insertions(+), 277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 43d292a..63243fb 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -55,6 +55,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3390. Reuse TimelineCollectorManager for RM (Zhijie Shen via sjlee)
+ YARN-3431. Sub resources of timeline entity needs to be passed to a separate
+ endpoint. (Zhijie Shen via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
index 9dc0c1d..734c741 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
@@ -20,16 +20,17 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement(name = "appattempt")
-@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ApplicationAttemptEntity extends HierarchicalTimelineEntity {
public ApplicationAttemptEntity() {
super(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString());
}
+
+ public ApplicationAttemptEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
index 45ec520..183d8d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
@@ -20,28 +20,28 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement(name = "application")
-@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ApplicationEntity extends HierarchicalTimelineEntity {
- private String queue;
+ public static final String QUEUE_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "QUEUE";
public ApplicationEntity() {
super(TimelineEntityType.YARN_APPLICATION.toString());
}
- @XmlElement(name = "queue")
+ public ApplicationEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
+
public String getQueue() {
- return queue;
+ return getInfo().get(QUEUE_INFO_KEY).toString();
}
public void setQueue(String queue) {
- this.queue = queue;
+ addInfo(QUEUE_INFO_KEY, queue);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
index a4278c0..94eefa8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
@@ -20,12 +20,6 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement(name = "cluster")
-@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ClusterEntity extends HierarchicalTimelineEntity {
@@ -33,4 +27,10 @@ public class ClusterEntity extends HierarchicalTimelineEntity {
super(TimelineEntityType.YARN_CLUSTER.toString());
}
+ public ClusterEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_CLUSTER.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
index cde6040..c7d6bce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
@@ -20,16 +20,17 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement(name = "container")
-@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ContainerEntity extends HierarchicalTimelineEntity {
public ContainerEntity() {
super(TimelineEntityType.YARN_CONTAINER.toString());
}
+
+ public ContainerEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_CONTAINER.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
index 0765f00..4554778 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowEntity.java
@@ -20,62 +20,84 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-@XmlRootElement(name = "flow")
-@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class FlowEntity extends HierarchicalTimelineEntity {
- private String user;
- private String version;
- private String run;
+ public static final String USER_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "USER";
+ public static final String FLOW_NAME_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_NAME";
+ public static final String FLOW_VERSION_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_VERSION";
+ public static final String FLOW_RUN_ID_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "FLOW_RUN_ID";
public FlowEntity() {
super(TimelineEntityType.YARN_FLOW.toString());
}
+ public FlowEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_FLOW.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
+
+ @XmlElement(name = "id")
@Override
public String getId() {
- //Flow id schema: user@flow_name(or id)/version/run
- StringBuilder sb = new StringBuilder();
- sb.append(user);
- sb.append('@');
- sb.append(super.getId());
- sb.append('/');
- sb.append(version);
- sb.append('/');
- sb.append(run);
- return sb.toString();
+ //Flow id schema: user@flow_name(or id)/version/run_id
+ String id = super.getId();
+ if (id == null) {
+ StringBuilder sb = new StringBuilder();
+ sb.append(getInfo().get(USER_INFO_KEY).toString());
+ sb.append('@');
+ sb.append(getInfo().get(FLOW_NAME_INFO_KEY).toString());
+ sb.append('/');
+ sb.append(getInfo().get(FLOW_VERSION_INFO_KEY).toString());
+ sb.append('/');
+ sb.append(getInfo().get(FLOW_RUN_ID_INFO_KEY).toString());
+ id = sb.toString();
+ setId(id);
+ }
+ return id;
}
- @XmlElement(name = "user")
public String getUser() {
- return user;
+ Object user = getInfo().get(USER_INFO_KEY);
+ return user == null ? null : user.toString();
}
public void setUser(String user) {
- this.user = user;
+ addInfo(USER_INFO_KEY, user);
+ }
+
+ public String getName() {
+ Object name = getInfo().get(FLOW_NAME_INFO_KEY);
+ return name == null ? null : name.toString();
+ }
+
+ public void setName(String name) {
+ addInfo(FLOW_NAME_INFO_KEY, name);
}
- @XmlElement(name = "version")
public String getVersion() {
- return version;
+ Object version = getInfo().get(FLOW_VERSION_INFO_KEY);
+ return version == null ? null : version.toString();
}
public void setVersion(String version) {
- this.version = version;
+ addInfo(FLOW_VERSION_INFO_KEY, version);
}
- @XmlElement(name = "run")
- public String getRun() {
- return run;
+ public long getRunId() {
+ Object runId = getInfo().get(FLOW_RUN_ID_INFO_KEY);
+ return runId == null ? 0L : (Long) runId;
}
- public void setRun(String run) {
- this.run = run;
+ public void setRunId(long runId) {
+ addInfo(FLOW_RUN_ID_INFO_KEY, runId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
index 49576de..6235da0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
@@ -17,93 +17,98 @@
*/
package org.apache.hadoop.yarn.api.records.timelineservice;
+import com.google.common.base.Joiner;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import java.util.HashMap;
+import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashSet;
-import java.util.Map;
+import java.util.List;
import java.util.Set;
-@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public abstract class HierarchicalTimelineEntity extends TimelineEntity {
- private Identifier parent;
- private HashMap<String, Set<String>> children = new HashMap<>();
+ public static final String PARENT_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "PARENT_ENTITY";
+ public static final String CHILDREN_INFO_KEY =
+ TimelineEntity.SYSTEM_INFO_KEY_PREFIX + "CHILDREN_ENTITY";
+
+ HierarchicalTimelineEntity(TimelineEntity entity) {
+ super(entity);
+ }
HierarchicalTimelineEntity(String type) {
super(type);
}
- @XmlElement(name = "parent")
public Identifier getParent() {
- return parent;
+ Object obj = getInfo().get(PARENT_INFO_KEY);
+ if (obj != null) {
+ if (obj instanceof Identifier) {
+ return (Identifier) obj;
+ } else {
+ throw new YarnRuntimeException(
+ "Parent info is invalid identifier object");
+ }
+ }
+ return null;
}
public void setParent(Identifier parent) {
validateParent(parent.getType());
- this.parent = parent;
+ addInfo(PARENT_INFO_KEY, parent);
}
public void setParent(String type, String id) {
- validateParent(type);
- parent = new Identifier();
- parent.setType(type);
- parent.setId(id);
+ setParent(new Identifier(type, id));
}
- // required by JAXB
- @InterfaceAudience.Private
- // comment out XmlElement here because it cause UnrecognizedPropertyException
- // TODO we need a better fix
- //@XmlElement(name = "children")
- public HashMap<String, Set<String>> getChildrenJAXB() {
+ public Set<Identifier> getChildren() {
+ Object identifiers = getInfo().get(CHILDREN_INFO_KEY);
+ if (identifiers == null) {
+ return new HashSet<>();
+ }
+ TimelineEntityType thisType = TimelineEntityType.valueOf(getType());
+ if (identifiers instanceof Set<?>) {
+ for (Object identifier : (Set<?>) identifiers) {
+ if (!(identifier instanceof Identifier)) {
+ throw new YarnRuntimeException(
+ "Children info contains invalid identifier object");
+ } else {
+ validateChild((Identifier) identifier, thisType);
+ }
+ }
+ } else {
+ throw new YarnRuntimeException(
+ "Children info is invalid identifier set");
+ }
+ Set<Identifier> children = (Set<Identifier>) identifiers;
return children;
}
- public Map<String, Set<String>> getChildren() {
- return children;
+ public void setChildren(Set<Identifier> children) {
+ addInfo(CHILDREN_INFO_KEY, children);
}
- public void setChildren(Map<String, Set<String>> children) {
- validateChildren(children);
- if (children != null && !(children instanceof HashMap)) {
- this.children = new HashMap<String, Set<String>>(children);
- } else {
- this.children = (HashMap) children;
+ public void addChildren(Set<Identifier> children) {
+ TimelineEntityType thisType = TimelineEntityType.valueOf(getType());
+ for (Identifier child : children) {
+ validateChild(child, thisType);
}
+ Set<Identifier> existingChildren = getChildren();
+ existingChildren.addAll(children);
+ setChildren(existingChildren);
}
- public void addChildren(Map<String, Set<String>> children) {
- validateChildren(children);
- for (Map.Entry<String, Set<String>> entry : children.entrySet()) {
- Set<String> ids = this.children.get(entry.getKey());
- if (ids == null) {
- ids = new HashSet<>();
- this.children.put(entry.getKey(), ids);
- }
- ids.addAll(entry.getValue());
- }
+ public void addChild(Identifier child) {
+ addChildren(Collections.singleton(child));
}
public void addChild(String type, String id) {
- TimelineEntityType thisType = TimelineEntityType.valueOf(getType());
- TimelineEntityType childType = TimelineEntityType.valueOf(type);
- if (thisType.isChild(childType)) {
- Set<String> ids = children.get(type);
- if (ids == null) {
- ids = new HashSet<>();
- children.put(type, ids);
- }
- ids.add(id);
- } else {
- throw new IllegalArgumentException(
- type + " is not the acceptable child of " + this.getType());
- }
+ addChild(new Identifier(type, id));
}
private void validateParent(String type) {
@@ -115,15 +120,12 @@ public abstract class HierarchicalTimelineEntity extends TimelineEntity {
}
}
- private void validateChildren(Map<String, Set<String>> children) {
- TimelineEntityType thisType = TimelineEntityType.valueOf(getType());
- for (Map.Entry<String, Set<String>> entry : children.entrySet()) {
- TimelineEntityType childType = TimelineEntityType.valueOf(entry.getKey());
- if (!thisType.isChild(childType)) {
- throw new IllegalArgumentException(
- entry.getKey() + " is not the acceptable child of " +
- this.getType());
- }
+ private void validateChild(Identifier child, TimelineEntityType thisType) {
+ TimelineEntityType childType = TimelineEntityType.valueOf(child.getType());
+ if (!thisType.isChild(childType)) {
+ throw new IllegalArgumentException(
+ child.getType() + " is not the acceptable child of " +
+ this.getType());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
new file mode 100644
index 0000000..d1a1f19
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class QueueEntity extends HierarchicalTimelineEntity {
+ public QueueEntity() {
+ super(TimelineEntityType.YARN_QUEUE.toString());
+ }
+
+ public QueueEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_QUEUE.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 1afb564..6cab753 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -34,6 +34,7 @@ import java.util.Set;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineEntity {
+ protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_";
@XmlRootElement(name = "identifier")
@XmlAccessorType(XmlAccessType.NONE)
@@ -41,6 +42,11 @@ public class TimelineEntity {
private String type;
private String id;
+ public Identifier(String type, String id) {
+ this.type = type;
+ this.id = id;
+ }
+
public Identifier() {
}
@@ -62,8 +68,16 @@ public class TimelineEntity {
public void setId(String id) {
this.id = id;
}
+
+ @Override
+ public String toString() {
+ return "TimelineEntity[" +
+ "type='" + type + '\'' +
+ ", id='" + id + '\'' + "]";
+ }
}
+ private TimelineEntity real;
private Identifier identifier;
private HashMap<String, Object> info = new HashMap<>();
private HashMap<String, Object> configs = new HashMap<>();
@@ -78,6 +92,22 @@ public class TimelineEntity {
identifier = new Identifier();
}
+ /**
+ * <p>
+ * The constuctor is used to construct a proxy {@link TimelineEntity} or its
+ * subclass object from the real entity object that carries information.
+ * </p>
+ *
+ * <p>
+ * It is usually used in the case where we want to recover class polymorphism
+ * after deserializing the entity from its JSON form.
+ * </p>
+ * @param entity the real entity that carries information
+ */
+ public TimelineEntity(TimelineEntity entity) {
+ real = entity.getReal();
+ }
+
protected TimelineEntity(String type) {
this();
identifier.type = type;
@@ -85,216 +115,378 @@ public class TimelineEntity {
@XmlElement(name = "type")
public String getType() {
- return identifier.type;
+ if (real == null) {
+ return identifier.type;
+ } else {
+ return real.getType();
+ }
}
public void setType(String type) {
- identifier.type = type;
+ if (real == null) {
+ identifier.type = type;
+ } else {
+ real.setType(type);
+ }
}
@XmlElement(name = "id")
public String getId() {
- return identifier.id;
+ if (real == null) {
+ return identifier.id;
+ } else {
+ return real.getId();
+ }
}
public void setId(String id) {
- identifier.id = id;
+ if (real == null) {
+ identifier.id = id;
+ } else {
+ real.setId(id);
+ }
}
public Identifier getIdentifier() {
- return identifier;
+ if (real == null) {
+ return identifier;
+ } else {
+ return real.getIdentifier();
+ }
}
public void setIdentifier(Identifier identifier) {
- this.identifier = identifier;
+ if (real == null) {
+ this.identifier = identifier;
+ } else {
+ real.setIdentifier(identifier);
+ }
}
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "info")
public HashMap<String, Object> getInfoJAXB() {
- return info;
+ if (real == null) {
+ return info;
+ } else {
+ return real.getInfoJAXB();
+ }
}
public Map<String, Object> getInfo() {
- return info;
+ if (real == null) {
+ return info;
+ } else {
+ return real.getInfo();
+ }
}
public void setInfo(Map<String, Object> info) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
+ if (real == null) {
+ if (info != null && !(info instanceof HashMap)) {
+ this.info = new HashMap<String, Object>(info);
+ } else {
+ this.info = (HashMap<String, Object>) info;
+ }
} else {
- this.info = (HashMap<String, Object>) info;
+ real.setInfo(info);
}
}
public void addInfo(Map<String, Object> info) {
- this.info.putAll(info);
+ if (real == null) {
+ this.info.putAll(info);
+ } else {
+ real.addInfo(info);
+ }
}
public void addInfo(String key, Object value) {
- info.put(key, value);
+ if (real == null) {
+ info.put(key, value);
+ } else {
+ real.addInfo(key, value);
+ }
}
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "configs")
public HashMap<String, Object> getConfigsJAXB() {
- return configs;
+ if (real == null) {
+ return configs;
+ } else {
+ return real.getConfigsJAXB();
+ }
}
public Map<String, Object> getConfigs() {
- return configs;
+ if (real == null) {
+ return configs;
+ } else {
+ return real.getConfigs();
+ }
}
public void setConfigs(Map<String, Object> configs) {
- if (configs != null && !(configs instanceof HashMap)) {
- this.configs = new HashMap<String, Object>(configs);
+ if (real == null) {
+ if (configs != null && !(configs instanceof HashMap)) {
+ this.configs = new HashMap<String, Object>(configs);
+ } else {
+ this.configs = (HashMap<String, Object>) configs;
+ }
} else {
- this.configs = (HashMap<String, Object>) configs;
+ real.setConfigs(configs);
}
}
public void addConfigs(Map<String, Object> configs) {
- this.configs.putAll(configs);
+ if (real == null) {
+ this.configs.putAll(configs);
+ } else {
+ real.addConfigs(configs);
+ }
}
public void addConfig(String key, Object value) {
- configs.put(key, value);
+ if (real == null) {
+ configs.put(key, value);
+ } else {
+ real.addConfig(key, value);
+ }
}
@XmlElement(name = "metrics")
public Set<TimelineMetric> getMetrics() {
- return metrics;
+ if (real == null) {
+ return metrics;
+ } else {
+ return real.getMetrics();
+ }
}
public void setMetrics(Set<TimelineMetric> metrics) {
- this.metrics = metrics;
+ if (real == null) {
+ this.metrics = metrics;
+ } else {
+ real.setMetrics(metrics);
+ }
}
public void addMetrics(Set<TimelineMetric> metrics) {
- this.metrics.addAll(metrics);
+ if (real == null) {
+ this.metrics.addAll(metrics);
+ } else {
+ real.addMetrics(metrics);
+ }
}
public void addMetric(TimelineMetric metric) {
- metrics.add(metric);
+ if (real == null) {
+ metrics.add(metric);
+ } else {
+ real.addMetric(metric);
+ }
}
@XmlElement(name = "events")
public Set<TimelineEvent> getEvents() {
- return events;
+ if (real == null) {
+ return events;
+ } else {
+ return real.getEvents();
+ }
}
public void setEvents(Set<TimelineEvent> events) {
- this.events = events;
+ if (real == null) {
+ this.events = events;
+ } else {
+ real.setEvents(events);
+ }
}
public void addEvents(Set<TimelineEvent> events) {
- this.events.addAll(events);
+ if (real == null) {
+ this.events.addAll(events);
+ } else {
+ real.addEvents(events);
+ }
}
public void addEvent(TimelineEvent event) {
- events.add(event);
+ if (real == null) {
+ events.add(event);
+ } else {
+ real.addEvent(event);
+ }
}
public Map<String, Set<String>> getIsRelatedToEntities() {
- return isRelatedToEntities;
+ if (real == null) {
+ return isRelatedToEntities;
+ } else {
+ return real.getIsRelatedToEntities();
+ }
}
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "isrelatedto")
public HashMap<String, Set<String>> getIsRelatedToEntitiesJAXB() {
- return isRelatedToEntities;
+ if (real == null) {
+ return isRelatedToEntities;
+ } else {
+ return real.getIsRelatedToEntitiesJAXB();
+ }
}
public void setIsRelatedToEntities(
Map<String, Set<String>> isRelatedToEntities) {
- if (isRelatedToEntities != null && !(isRelatedToEntities instanceof HashMap)) {
- this.isRelatedToEntities = new HashMap<String, Set<String>>(isRelatedToEntities);
+ if (real == null) {
+ if (isRelatedToEntities != null &&
+ !(isRelatedToEntities instanceof HashMap)) {
+ this.isRelatedToEntities =
+ new HashMap<String, Set<String>>(isRelatedToEntities);
+ } else {
+ this.isRelatedToEntities =
+ (HashMap<String, Set<String>>) isRelatedToEntities;
+ }
} else {
- this.isRelatedToEntities = (HashMap<String, Set<String>>) isRelatedToEntities;
+ real.setIsRelatedToEntities(isRelatedToEntities);
}
}
public void addIsRelatedToEntities(
Map<String, Set<String>> isRelatedToEntities) {
- for (Map.Entry<String, Set<String>> entry : isRelatedToEntities
- .entrySet()) {
- Set<String> ids = this.isRelatedToEntities.get(entry.getKey());
- if (ids == null) {
- ids = new HashSet<>();
- this.isRelatedToEntities.put(entry.getKey(), ids);
+ if (real == null) {
+ for (Map.Entry<String, Set<String>> entry : isRelatedToEntities
+ .entrySet()) {
+ Set<String> ids = this.isRelatedToEntities.get(entry.getKey());
+ if (ids == null) {
+ ids = new HashSet<>();
+ this.isRelatedToEntities.put(entry.getKey(), ids);
+ }
+ ids.addAll(entry.getValue());
}
- ids.addAll(entry.getValue());
+ } else {
+ real.addIsRelatedToEntities(isRelatedToEntities);
}
}
public void addIsRelatedToEntity(String type, String id) {
- Set<String> ids = isRelatedToEntities.get(type);
- if (ids == null) {
- ids = new HashSet<>();
- isRelatedToEntities.put(type, ids);
+ if (real == null) {
+ Set<String> ids = isRelatedToEntities.get(type);
+ if (ids == null) {
+ ids = new HashSet<>();
+ isRelatedToEntities.put(type, ids);
+ }
+ ids.add(id);
+ } else {
+ real.addIsRelatedToEntity(type, id);
}
- ids.add(id);
}
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "relatesto")
public HashMap<String, Set<String>> getRelatesToEntitiesJAXB() {
- return relatesToEntities;
+ if (real == null) {
+ return relatesToEntities;
+ } else {
+ return real.getRelatesToEntitiesJAXB();
+ }
}
public Map<String, Set<String>> getRelatesToEntities() {
- return relatesToEntities;
+ if (real == null) {
+ return relatesToEntities;
+ } else {
+ return real.getRelatesToEntities();
+ }
}
public void addRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
- for (Map.Entry<String, Set<String>> entry : relatesToEntities.entrySet()) {
- Set<String> ids = this.relatesToEntities.get(entry.getKey());
- if (ids == null) {
- ids = new HashSet<>();
- this.relatesToEntities.put(entry.getKey(), ids);
+ if (real == null) {
+ for (Map.Entry<String, Set<String>> entry : relatesToEntities
+ .entrySet()) {
+ Set<String> ids = this.relatesToEntities.get(entry.getKey());
+ if (ids == null) {
+ ids = new HashSet<>();
+ this.relatesToEntities.put(entry.getKey(), ids);
+ }
+ ids.addAll(entry.getValue());
}
- ids.addAll(entry.getValue());
+ } else {
+ real.addRelatesToEntities(relatesToEntities);
}
}
public void addRelatesToEntity(String type, String id) {
- Set<String> ids = relatesToEntities.get(type);
- if (ids == null) {
- ids = new HashSet<>();
- relatesToEntities.put(type, ids);
+ if (real == null) {
+ Set<String> ids = relatesToEntities.get(type);
+ if (ids == null) {
+ ids = new HashSet<>();
+ relatesToEntities.put(type, ids);
+ }
+ ids.add(id);
+ } else {
+ real.addRelatesToEntity(type, id);
}
- ids.add(id);
}
public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
- if (relatesToEntities != null && !(relatesToEntities instanceof HashMap)) {
- this.relatesToEntities = new HashMap<String, Set<String>>(relatesToEntities);
+ if (real == null) {
+ if (relatesToEntities != null &&
+ !(relatesToEntities instanceof HashMap)) {
+ this.relatesToEntities =
+ new HashMap<String, Set<String>>(relatesToEntities);
+ } else {
+ this.relatesToEntities =
+ (HashMap<String, Set<String>>) relatesToEntities;
+ }
} else {
- this.relatesToEntities = (HashMap<String, Set<String>>) relatesToEntities;
+ real.setRelatesToEntities(relatesToEntities);
}
}
@XmlElement(name = "createdtime")
public long getCreatedTime() {
- return createdTime;
+ if (real == null) {
+ return createdTime;
+ } else {
+ return real.getCreatedTime();
+ }
}
public void setCreatedTime(long createdTime) {
- this.createdTime = createdTime;
+ if (real == null) {
+ this.createdTime = createdTime;
+ } else {
+ real.setCreatedTime(createdTime);
+ }
}
@XmlElement(name = "modifiedtime")
public long getModifiedTime() {
- return modifiedTime;
+ if (real == null) {
+ return modifiedTime;
+ } else {
+ return real.getModifiedTime();
+ }
}
public void setModifiedTime(long modifiedTime) {
- this.modifiedTime = modifiedTime;
+ if (real == null) {
+ this.modifiedTime = modifiedTime;
+ } else {
+ real.setModifiedTime(modifiedTime);
+ }
}
+ protected TimelineEntity getReal() {
+ return real == null ? this : real;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java
deleted file mode 100644
index c4039f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineQueue.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.api.records.timelineservice;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement(name = "queue")
-@XmlAccessorType(XmlAccessType.NONE)
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineQueue extends HierarchicalTimelineEntity {
- public TimelineQueue() {
- super(TimelineEntityType.YARN_QUEUE.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java
deleted file mode 100644
index 45cf48f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineUser.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.api.records.timelineservice;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlRootElement;
-
-@XmlRootElement(name = "user")
-@XmlAccessorType(XmlAccessType.NONE)
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineUser extends TimelineEntity {
- public TimelineUser() {
- super(TimelineEntityType.YARN_USER.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
new file mode 100644
index 0000000..a229fd8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class UserEntity extends TimelineEntity {
+ public UserEntity() {
+ super(TimelineEntityType.YARN_USER.toString());
+ }
+
+ public UserEntity(TimelineEntity entity) {
+ super(entity);
+ if (!entity.getType().equals(TimelineEntityType.YARN_USER.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 4f8ab94..caa3f3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -23,8 +23,12 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
+import org.junit.Assert;
+
+import java.util.Collections;
public class TestTimelineServiceRecords {
@@ -87,10 +91,10 @@ public class TestTimelineServiceRecords {
@Test
public void testFirstClassCitizenEntities() throws Exception {
- TimelineUser user = new TimelineUser();
+ UserEntity user = new UserEntity();
user.setId("test user id");
- TimelineQueue queue = new TimelineQueue();
+ QueueEntity queue = new QueueEntity();
queue.setId("test queue id");
@@ -98,20 +102,26 @@ public class TestTimelineServiceRecords {
cluster.setId("test cluster id");
FlowEntity flow1 = new FlowEntity();
- flow1.setId("test flow id");
+ //flow1.setId("test flow id 1");
flow1.setUser(user.getId());
- flow1.setVersion("test flow version");
- flow1.setRun("test run 1");
+ flow1.setName("test flow name 1");
+ flow1.setVersion("test flow version 1");
+ flow1.setRunId(1L);
FlowEntity flow2 = new FlowEntity();
- flow2.setId("test flow run id2");
+ //flow2.setId("test flow run id 2");
flow2.setUser(user.getId());
- flow1.setVersion("test flow version2");
- flow2.setRun("test run 2");
+ flow2.setName("test flow name 2");
+ flow2.setVersion("test flow version 2");
+ flow2.setRunId(2L);
+
+ ApplicationEntity app1 = new ApplicationEntity();
+ app1.setId(ApplicationId.newInstance(0, 1).toString());
+ app1.setQueue(queue.getId());
- ApplicationEntity app = new ApplicationEntity();
- app.setId(ApplicationId.newInstance(0, 1).toString());
- app.setQueue(queue.getId());
+ ApplicationEntity app2 = new ApplicationEntity();
+ app2.setId(ApplicationId.newInstance(0, 2).toString());
+ app2.setQueue(queue.getId());
ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity();
appAttempt.setId(ApplicationAttemptId.newInstance(
@@ -127,12 +137,14 @@ public class TestTimelineServiceRecords {
.setParent(TimelineEntityType.YARN_CLUSTER.toString(), cluster.getId());
flow1.addChild(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
flow2.setParent(TimelineEntityType.YARN_FLOW.toString(), flow1.getId());
- flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app.getId());
- app.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
- app.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
+ flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId());
+ flow2.addChild(TimelineEntityType.YARN_APPLICATION.toString(), app2.getId());
+ app1.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
+ app1.addChild(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
appAttempt.getId());
appAttempt
- .setParent(TimelineEntityType.YARN_APPLICATION.toString(), app.getId());
+ .setParent(TimelineEntityType.YARN_APPLICATION.toString(), app1.getId());
+ app2.setParent(TimelineEntityType.YARN_FLOW.toString(), flow2.getId());
appAttempt.addChild(TimelineEntityType.YARN_CONTAINER.toString(),
container.getId());
container.setParent(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
@@ -141,14 +153,57 @@ public class TestTimelineServiceRecords {
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(cluster, true));
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(flow1, true));
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(flow2, true));
- LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(app, true));
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(app1, true));
+ LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(app2, true));
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(appAttempt, true));
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(container, true));
+
+
+ // Check parent/children APIs
+ Assert.assertNotNull(app1.getParent());
+ Assert.assertEquals(flow2.getType(), app1.getParent().getType());
+ Assert.assertEquals(flow2.getId(), app1.getParent().getId());
+ app1.addInfo(ApplicationEntity.PARENT_INFO_KEY, "invalid parent object");
+ try {
+ app1.getParent();
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof YarnRuntimeException);
+ Assert.assertTrue(e.getMessage().contains(
+ "Parent info is invalid identifier object"));
+ }
+
+ Assert.assertNotNull(app1.getChildren());
+ Assert.assertEquals(1, app1.getChildren().size());
+ Assert.assertEquals(
+ appAttempt.getType(), app1.getChildren().iterator().next().getType());
+ Assert.assertEquals(
+ appAttempt.getId(), app1.getChildren().iterator().next().getId());
+ app1.addInfo(ApplicationEntity.CHILDREN_INFO_KEY,
+ Collections.singletonList("invalid children set"));
+ try {
+ app1.getChildren();
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof YarnRuntimeException);
+ Assert.assertTrue(e.getMessage().contains(
+ "Children info is invalid identifier set"));
+ }
+ app1.addInfo(ApplicationEntity.CHILDREN_INFO_KEY,
+ Collections.singleton("invalid child object"));
+ try {
+ app1.getChildren();
+ Assert.fail();
+ } catch (Exception e) {
+ Assert.assertTrue(e instanceof YarnRuntimeException);
+ Assert.assertTrue(e.getMessage().contains(
+ "Children info contains invalid identifier object"));
+ }
}
@Test
public void testUser() throws Exception {
- TimelineUser user = new TimelineUser();
+ UserEntity user = new UserEntity();
user.setId("test user id");
user.addInfo("test info key 1", "test info value 1");
user.addInfo("test info key 2", "test info value 2");
@@ -157,7 +212,7 @@ public class TestTimelineServiceRecords {
@Test
public void testQueue() throws Exception {
- TimelineQueue queue = new TimelineQueue();
+ QueueEntity queue = new QueueEntity();
queue.setId("test queue id");
queue.addInfo("test info key 1", "test info value 1");
queue.addInfo("test info key 2", "test info value 2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 0bdb68a..044148a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -24,9 +24,12 @@ import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.*;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -85,6 +88,45 @@ public class TestTimelineServiceClientIntegration {
}
}
+ @Test
+ public void testPutExtendedEntities() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ TimelineClient client =
+ TimelineClient.createTimelineClient(appId);
+ try {
+ // set the timeline service address manually
+ client.setTimelineServiceAddress(
+ collectorManager.getRestServerBindAddress());
+ client.init(new YarnConfiguration());
+ client.start();
+ ClusterEntity cluster = new ClusterEntity();
+ cluster.setId(YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
+ FlowEntity flow = new FlowEntity();
+ flow.setUser(UserGroupInformation.getCurrentUser().getShortUserName());
+ flow.setName("test_flow_name");
+ flow.setVersion("test_flow_version");
+ flow.setRunId(1L);
+ flow.setParent(cluster.getType(), cluster.getId());
+ ApplicationEntity app = new ApplicationEntity();
+ app.setId(appId.toString());
+ flow.addChild(app.getType(), app.getId());
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ ApplicationAttemptEntity appAttempt = new ApplicationAttemptEntity();
+ appAttempt.setId(attemptId.toString());
+ ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
+ ContainerEntity container = new ContainerEntity();
+ container.setId(containerId.toString());
+ UserEntity user = new UserEntity();
+ user.setId(UserGroupInformation.getCurrentUser().getShortUserName());
+ QueueEntity queue = new QueueEntity();
+ queue.setId("default_queue");
+ client.putEntities(cluster, flow, app, appAttempt, container, user, queue);
+ client.putEntitiesAsync(cluster, flow, app, appAttempt, container, user, queue);
+ } finally {
+ client.stop();
+ }
+ }
+
private static class MockNodeTimelineCollectorManager extends
NodeTimelineCollectorManager {
public MockNodeTimelineCollectorManager() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8fd1aa15/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index 2165c66..42fa365 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -43,7 +43,16 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.UserEntity;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
@@ -142,7 +151,8 @@ public class TimelineCollectorWebService {
LOG.error("Application: "+ appId + " is not found");
throw new NotFoundException(); // different exception?
}
- collector.putEntities(entities, callerUgi);
+
+ collector.putEntities(processTimelineEntities(entities), callerUgi);
return Response.ok().build();
} catch (Exception e) {
LOG.error("Error putting entities", e);
@@ -151,7 +161,7 @@ public class TimelineCollectorWebService {
}
}
- private ApplicationId parseApplicationId(String appId) {
+ private static ApplicationId parseApplicationId(String appId) {
try {
if (appId != null) {
return ConverterUtils.toApplicationId(appId.trim());
@@ -159,15 +169,16 @@ public class TimelineCollectorWebService {
return null;
}
} catch (Exception e) {
+ LOG.error("Invalid application ID: " + appId);
return null;
}
}
- private void init(HttpServletResponse response) {
+ private static void init(HttpServletResponse response) {
response.setContentType(null);
}
- private UserGroupInformation getUser(HttpServletRequest req) {
+ private static UserGroupInformation getUser(HttpServletRequest req) {
String remoteUser = req.getRemoteUser();
UserGroupInformation callerUgi = null;
if (remoteUser != null) {
@@ -175,4 +186,50 @@ public class TimelineCollectorWebService {
}
return callerUgi;
}
+
+ // The process may not be necessary according to the way we write the backend,
+ // but let's keep it for now in case we need to use sub-classes APIs in the
+ // future (e.g., aggregation).
+ private static TimelineEntities processTimelineEntities(
+ TimelineEntities entities) {
+ TimelineEntities entitiesToReturn = new TimelineEntities();
+ for (TimelineEntity entity : entities.getEntities()) {
+ TimelineEntityType type = null;
+ try {
+ type = TimelineEntityType.valueOf(entity.getType());
+ } catch (IllegalArgumentException e) {
+ type = null;
+ }
+ if (type != null) {
+ switch (type) {
+ case YARN_CLUSTER:
+ entitiesToReturn.addEntity(new ClusterEntity(entity));
+ break;
+ case YARN_FLOW:
+ entitiesToReturn.addEntity(new FlowEntity(entity));
+ break;
+ case YARN_APPLICATION:
+ entitiesToReturn.addEntity(new ApplicationEntity(entity));
+ break;
+ case YARN_APPLICATION_ATTEMPT:
+ entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity));
+ break;
+ case YARN_CONTAINER:
+ entitiesToReturn.addEntity(new ContainerEntity(entity));
+ break;
+ case YARN_QUEUE:
+ entitiesToReturn.addEntity(new QueueEntity(entity));
+ break;
+ case YARN_USER:
+ entitiesToReturn.addEntity(new UserEntity(entity));
+ break;
+ default:
+ break;
+ }
+ } else {
+ entitiesToReturn.addEntity(entity);
+ }
+ }
+ return entitiesToReturn;
+ }
}
[06/50] [abbrv] hadoop git commit: YARN-3333. Rename
TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
new file mode 100644
index 0000000..009fa63
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.nodemanager.collectormanager;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+
+public class NMCollectorService extends CompositeService implements
+ CollectorNodemanagerProtocol {
+
+ private static final Log LOG = LogFactory.getLog(NMCollectorService.class);
+
+ final Context context;
+
+ private Server server;
+
+ public NMCollectorService(Context context) {
+
+ super(NMCollectorService.class.getName());
+ this.context = context;
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ Configuration conf = getConfig();
+
+ InetSocketAddress collectorServerAddress = conf.getSocketAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
+
+ Configuration serverConf = new Configuration(conf);
+
+ // TODO Security settings.
+ YarnRPC rpc = YarnRPC.create(conf);
+
+ server =
+ rpc.getServer(CollectorNodemanagerProtocol.class, this,
+ collectorServerAddress, serverConf,
+ this.context.getNMTokenSecretManager(),
+ conf.getInt(YarnConfiguration.NM_COLLECTOR_SERVICE_THREAD_COUNT,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT));
+
+ server.start();
+ // start remaining services
+ super.serviceStart();
+ LOG.info("NMCollectorService started at " + collectorServerAddress);
+ }
+
+
+ @Override
+ public void serviceStop() throws Exception {
+ if (server != null) {
+ server.stop();
+ }
+ // TODO may cleanup app collectors running on this NM in future.
+ super.serviceStop();
+ }
+
+ @Override
+ public ReportNewCollectorInfoResponse reportNewCollectorInfo(
+ ReportNewCollectorInfoRequest request) throws IOException {
+ List<AppCollectorsMap> newCollectorsList = request.getAppCollectorsList();
+ if (newCollectorsList != null && !newCollectorsList.isEmpty()) {
+ Map<ApplicationId, String> newCollectorsMap =
+ new HashMap<ApplicationId, String>();
+ for (AppCollectorsMap collector : newCollectorsList) {
+ newCollectorsMap.put(collector.getApplicationId(), collector.getCollectorAddr());
+ }
+ ((NodeManager.NMContext)context).addRegisteredCollectors(newCollectorsMap);
+ }
+
+ return ReportNewCollectorInfoResponse.newInstance();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index ba5033f..cc82401 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -424,10 +424,11 @@ public class ApplicationImpl implements Application {
new LogHandlerAppFinishedEvent(app.appId));
app.context.getNMTokenSecretManager().appFinished(app.getAppId());
- // Remove aggregator info for finished apps.
- // TODO check we remove related aggregators info in failure cases (YARN-3038)
- app.context.getRegisteredAggregators().remove(app.getAppId());
- app.context.getKnownAggregators().remove(app.getAppId());
+ // Remove collectors info for finished apps.
+ // TODO check we remove related collectors info in failure cases
+ // (YARN-3038)
+ app.context.getRegisteredCollectors().remove(app.getAppId());
+ app.context.getKnownCollectors().remove(app.getAppId());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 30c9200..db7eda2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -351,8 +351,8 @@ public class ApplicationMasterService extends AbstractService implements
RMApp rmApp =
rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
- // Remove aggregator address when app get finished.
- rmApp.removeAggregatorAddr();
+ // Remove collector address when app get finished.
+ rmApp.removeCollectorAddr();
// checking whether the app exits in RMStateStore at first not to throw
// ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart.
@@ -601,10 +601,10 @@ public class ApplicationMasterService extends AbstractService implements
allocateResponse.setAvailableResources(allocation.getResourceLimit());
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
-
- // add aggregator address for this application
- allocateResponse.setAggregatorAddr(
- this.rmContext.getRMApps().get(applicationId).getAggregatorAddr());
+
+ // add collector address for this application
+ allocateResponse.setCollectorAddr(
+ this.rmContext.getRMApps().get(applicationId).getCollectorAddr());
// add preemption to the allocateResponse message (if any)
allocateResponse
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 052655f..e98fefc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -65,7 +65,7 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppAggregatorUpdateEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppCollectorUpdateEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -445,11 +445,11 @@ public class ResourceTrackerService extends AbstractService implements
return YarnServerBuilderUtils.newNodeHeartbeatResponse(NodeAction.RESYNC,
message);
}
-
- // Check & update aggregators info from request.
+
+ // Check & update collectors info from request.
// TODO make sure it won't have race condition issue for AM failed over case
// that the older registration could possible override the newer one.
- updateAppAggregatorsMap(request);
+ updateAppCollectorsMap(request);
// Heartbeat response
NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
@@ -465,13 +465,14 @@ public class ResourceTrackerService extends AbstractService implements
if (!systemCredentials.isEmpty()) {
nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
}
-
- // Return aggregators' map that NM needs to know
- // TODO we should optimize this to only include aggreator info that NM
+
+ // Return collectors' map that NM needs to know
+ // TODO we should optimize this to only include collector info that NM
// doesn't know yet.
- List<ApplicationId> keepAliveApps = remoteNodeStatus.getKeepAliveApplications();
+ List<ApplicationId> keepAliveApps =
+ remoteNodeStatus.getKeepAliveApplications();
if (keepAliveApps != null) {
- setAppAggregatorsMapToResponse(keepAliveApps, nodeHeartBeatResponse);
+ setAppCollectorsMapToResponse(keepAliveApps, nodeHeartBeatResponse);
}
// 4. Send status to RMNode, saving the latest response.
@@ -501,48 +502,49 @@ public class ResourceTrackerService extends AbstractService implements
return nodeHeartBeatResponse;
}
-
- private void setAppAggregatorsMapToResponse(
+
+ private void setAppCollectorsMapToResponse(
List<ApplicationId> liveApps, NodeHeartbeatResponse response) {
- Map<ApplicationId, String> liveAppAggregatorsMap = new
+ Map<ApplicationId, String> liveAppCollectorsMap = new
ConcurrentHashMap<ApplicationId, String>();
Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps();
for (ApplicationId appId : liveApps) {
- String appAggregatorAddr = rmApps.get(appId).getAggregatorAddr();
- if (appAggregatorAddr != null) {
- liveAppAggregatorsMap.put(appId, appAggregatorAddr);
+ String appCollectorAddr = rmApps.get(appId).getCollectorAddr();
+ if (appCollectorAddr != null) {
+ liveAppCollectorsMap.put(appId, appCollectorAddr);
} else {
- // Log a debug info if aggregator address is not found.
+ // Log a debug info if collector address is not found.
if (LOG.isDebugEnabled()) {
- LOG.debug("Aggregator for applicaton: " + appId + " hasn't registered yet!");
+ LOG.debug("Collector for applicaton: " + appId +
+ " hasn't registered yet!");
}
}
}
- response.setAppAggregatorsMap(liveAppAggregatorsMap);
+ response.setAppCollectorsMap(liveAppCollectorsMap);
}
-
- private void updateAppAggregatorsMap(NodeHeartbeatRequest request) {
- Map<ApplicationId, String> registeredAggregatorsMap =
- request.getRegisteredAggregators();
- if (registeredAggregatorsMap != null
- && !registeredAggregatorsMap.isEmpty()) {
+
+ private void updateAppCollectorsMap(NodeHeartbeatRequest request) {
+ Map<ApplicationId, String> registeredCollectorsMap =
+ request.getRegisteredCollectors();
+ if (registeredCollectorsMap != null
+ && !registeredCollectorsMap.isEmpty()) {
Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps();
- for (Map.Entry<ApplicationId, String> entry:
- registeredAggregatorsMap.entrySet()) {
+ for (Map.Entry<ApplicationId, String> entry:
+ registeredCollectorsMap.entrySet()) {
ApplicationId appId = entry.getKey();
- String aggregatorAddr = entry.getValue();
- if (aggregatorAddr != null && !aggregatorAddr.isEmpty()) {
+ String collectorAddr = entry.getValue();
+ if (collectorAddr != null && !collectorAddr.isEmpty()) {
RMApp rmApp = rmApps.get(appId);
if (rmApp == null) {
- LOG.warn("Cannot update aggregator info because application ID: " +
+ LOG.warn("Cannot update collector info because application ID: " +
appId + " is not found in RMContext!");
} else {
- String previousAggregatorAddr = rmApp.getAggregatorAddr();
- if (previousAggregatorAddr == null ||
- previousAggregatorAddr != aggregatorAddr) {
- // sending aggregator update event.
- RMAppAggregatorUpdateEvent event =
- new RMAppAggregatorUpdateEvent(appId, aggregatorAddr);
+ String previousCollectorAddr = rmApp.getCollectorAddr();
+ if (previousCollectorAddr == null ||
+ previousCollectorAddr != collectorAddr) {
+ // sending collector update event.
+ RMAppCollectorUpdateEvent event =
+ new RMAppCollectorUpdateEvent(appId, collectorAddr);
rmContext.getDispatcher().getEventHandler().handle(event);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 8fbedb2..d4b758e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -174,23 +174,23 @@ public interface RMApp extends EventHandler<RMAppEvent> {
* @return the tracking url for the application master.
*/
String getTrackingUrl();
-
+
/**
- * The aggregator address for the application.
- * @return the address for the application's aggregator.
+ * The collector address for the application.
+ * @return the address for the application's collector.
*/
- String getAggregatorAddr();
-
+ String getCollectorAddr();
+
/**
- * Set aggregator address for the application
- * @param aggregatorAddr the address of aggregator
+ * Set collector address for the application
+ * @param collectorAddr the address of collector
*/
- void setAggregatorAddr(String aggregatorAddr);
-
+ void setCollectorAddr(String collectorAddr);
+
/**
- * Remove aggregator address when application is finished or killed.
+ * Remove collector address when application is finished or killed.
*/
- void removeAggregatorAddr();
+ void removeCollectorAddr();
/**
* The original tracking url for the application master.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java
deleted file mode 100644
index b43de44..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppAggregatorUpdateEvent.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-
-public class RMAppAggregatorUpdateEvent extends RMAppEvent {
-
- private final String appAggregatorAddr;
-
- public RMAppAggregatorUpdateEvent(ApplicationId appId, String appAggregatorAddr) {
- super(appId, RMAppEventType.AGGREGATOR_UPDATE);
- this.appAggregatorAddr = appAggregatorAddr;
- }
-
- public String getAppAggregatorAddr(){
- return this.appAggregatorAddr;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
new file mode 100644
index 0000000..698c9b5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+public class RMAppCollectorUpdateEvent extends RMAppEvent {
+
+ private final String appCollectorAddr;
+
+ public RMAppCollectorUpdateEvent(ApplicationId appId,
+ String appCollectorAddr) {
+ super(appId, RMAppEventType.COLLECTOR_UPDATE);
+ this.appCollectorAddr = appCollectorAddr;
+ }
+
+ public String getAppCollectorAddr(){
+ return this.appCollectorAddr;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
index 6e9460a..2b42638 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppEventType.java
@@ -30,9 +30,9 @@ public enum RMAppEventType {
// Source: Scheduler
APP_ACCEPTED,
-
+
// TODO add source later
- AGGREGATOR_UPDATE,
+ COLLECTOR_UPDATE,
// Source: RMAppAttempt
ATTEMPT_REGISTERED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 6536783..c4f5270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -142,7 +142,7 @@ public class RMAppImpl implements RMApp, Recoverable {
private long startTime;
private long finishTime = 0;
private long storedFinishTime = 0;
- private String aggregatorAddr;
+ private String collectorAddr;
// This field isn't protected by readlock now.
private volatile RMAppAttempt currentAttempt;
private String queue;
@@ -189,7 +189,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.NEW, RMAppState.NEW,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.NEW, RMAppState.NEW,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.NEW, RMAppState.NEW_SAVING,
RMAppEventType.START, new RMAppNewlySavingTransition())
.addTransition(RMAppState.NEW, EnumSet.of(RMAppState.SUBMITTED,
@@ -207,7 +207,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
RMAppEventType.NODE_UPDATE, new RMAppNodeUpdateTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.NEW_SAVING,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.SUBMITTED,
RMAppEventType.APP_NEW_SAVED, new AddApplicationToSchedulerTransition())
.addTransition(RMAppState.NEW_SAVING, RMAppState.FINAL_SAVING,
@@ -227,7 +227,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.SUBMITTED, RMAppState.SUBMITTED,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.SUBMITTED, RMAppState.FINAL_SAVING,
RMAppEventType.APP_REJECTED,
new FinalSavingTransition(
@@ -245,7 +245,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.ACCEPTED, RMAppState.ACCEPTED,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.ACCEPTED, RMAppState.RUNNING,
RMAppEventType.ATTEMPT_REGISTERED)
.addTransition(RMAppState.ACCEPTED,
@@ -273,7 +273,7 @@ public class RMAppImpl implements RMApp, Recoverable {
.addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
RMAppEventType.MOVE, new RMAppMoveTransition())
.addTransition(RMAppState.RUNNING, RMAppState.RUNNING,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.RUNNING, RMAppState.FINAL_SAVING,
RMAppEventType.ATTEMPT_UNREGISTERED,
new FinalSavingTransition(
@@ -304,7 +304,7 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition())
.addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
// ignorable transitions
.addTransition(RMAppState.FINAL_SAVING, RMAppState.FINAL_SAVING,
EnumSet.of(RMAppEventType.NODE_UPDATE, RMAppEventType.KILL,
@@ -317,7 +317,7 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition())
.addTransition(RMAppState.FINISHING, RMAppState.FINISHING,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
// ignorable transitions
.addTransition(RMAppState.FINISHING, RMAppState.FINISHING,
EnumSet.of(RMAppEventType.NODE_UPDATE,
@@ -330,7 +330,7 @@ public class RMAppImpl implements RMApp, Recoverable {
RMAppEventType.APP_RUNNING_ON_NODE,
new AppRunningOnNodeTransition())
.addTransition(RMAppState.KILLING, RMAppState.KILLING,
- RMAppEventType.AGGREGATOR_UPDATE, new RMAppAggregatorUpdateTransition())
+ RMAppEventType.COLLECTOR_UPDATE, new RMAppCollectorUpdateTransition())
.addTransition(RMAppState.KILLING, RMAppState.FINAL_SAVING,
RMAppEventType.ATTEMPT_KILLED,
new FinalSavingTransition(
@@ -548,20 +548,20 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setQueue(String queue) {
this.queue = queue;
}
-
+
@Override
- public String getAggregatorAddr() {
- return this.aggregatorAddr;
+ public String getCollectorAddr() {
+ return this.collectorAddr;
}
-
+
@Override
- public void setAggregatorAddr(String aggregatorAddr) {
- this.aggregatorAddr = aggregatorAddr;
+ public void setCollectorAddr(String collectorAddr) {
+ this.collectorAddr = collectorAddr;
}
-
+
@Override
- public void removeAggregatorAddr() {
- this.aggregatorAddr = null;
+ public void removeCollectorAddr() {
+ this.collectorAddr = null;
}
@Override
@@ -818,8 +818,8 @@ public class RMAppImpl implements RMApp, Recoverable {
.getDiagnostics());
this.storedFinishTime = appState.getFinishTime();
this.startTime = appState.getStartTime();
- //TODO recover aggregator address.
- //this.aggregatorAddr = appState.getAggregatorAddr();
+ //TODO recover collector address.
+ //this.collectorAddr = appState.getCollectorAddr();
for(int i=0; i<appState.getAttemptCount(); ++i) {
// create attempt
@@ -863,22 +863,22 @@ public class RMAppImpl implements RMApp, Recoverable {
};
}
- private static final class RMAppAggregatorUpdateTransition
+ private static final class RMAppCollectorUpdateTransition
extends RMAppTransition {
-
+
public void transition(RMAppImpl app, RMAppEvent event) {
- LOG.info("Updating aggregator info for app: " + app.getApplicationId());
-
- RMAppAggregatorUpdateEvent appAggregatorUpdateEvent =
- (RMAppAggregatorUpdateEvent) event;
- // Update aggregator address
- app.setAggregatorAddr(appAggregatorUpdateEvent.getAppAggregatorAddr());
-
+ LOG.info("Updating collector info for app: " + app.getApplicationId());
+
+ RMAppCollectorUpdateEvent appCollectorUpdateEvent =
+ (RMAppCollectorUpdateEvent) event;
+ // Update collector address
+ app.setCollectorAddr(appCollectorUpdateEvent.getAppCollectorAddr());
+
// TODO persistent to RMStateStore for recover
// Save to RMStateStore
};
}
-
+
private static final class RMAppNodeUpdateTransition extends RMAppTransition {
public void transition(RMAppImpl app, RMAppEvent event) {
RMAppNodeUpdateEvent nodeUpdateEvent = (RMAppNodeUpdateEvent) event;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index d9cdee0..e52b054 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -94,15 +94,15 @@ public abstract class MockAsm extends MockApps {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public String getAggregatorAddr() {
+ public String getCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public void setAggregatorAddr(String aggregatorAddr) {
+ public void setCollectorAddr(String collectorAddr) {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public void removeAggregatorAddr() {
+ public void removeCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 83e99b8..aec5379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -285,17 +285,17 @@ public class MockRMApp implements RMApp {
}
@Override
- public String getAggregatorAddr() {
+ public String getCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet.");
}
-
+
@Override
- public void removeAggregatorAddr() {
+ public void removeCollectorAddr() {
throw new UnsupportedOperationException("Not supported yet.");
}
@Override
- public void setAggregatorAddr(String aggregatorAddr) {
+ public void setCollectorAddr(String collectorAddr) {
throw new UnsupportedOperationException("Not supported yet.");
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 3b7fba0..e04e858 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -56,6 +56,11 @@
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 32ee5d8..fab131c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -1,25 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.yarn.server.timelineservice;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.aggregator.PerNodeTimelineAggregatorsAuxService;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import static org.junit.Assert.fail;
-
public class TestTimelineServiceClientIntegration {
- private static PerNodeTimelineAggregatorsAuxService auxService;
+ private static TimelineCollectorManager collectorManager;
+ private static PerNodeTimelineCollectorsAuxService auxService;
@BeforeClass
public static void setupClass() throws Exception {
try {
- auxService = PerNodeTimelineAggregatorsAuxService.launchServer(new String[0]);
+ collectorManager = new MyTimelineCollectorManager();
+ auxService =
+ PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
+ collectorManager);
auxService.addApplication(ApplicationId.newInstance(0, 1));
} catch (ExitUtil.ExitException e) {
fail();
@@ -38,6 +63,9 @@ public class TestTimelineServiceClientIntegration {
TimelineClient client =
TimelineClient.createTimelineClient(ApplicationId.newInstance(0, 1));
try {
+ // set the timeline service address manually
+ client.setTimelineServiceAddress(
+ collectorManager.getRestServerBindAddress());
client.init(new YarnConfiguration());
client.start();
TimelineEntity entity = new TimelineEntity();
@@ -45,10 +73,20 @@ public class TestTimelineServiceClientIntegration {
entity.setId("test entity id");
client.putEntities(entity);
client.putEntitiesAsync(entity);
- } catch(Exception e) {
- fail();
} finally {
client.stop();
}
}
+
+ private static class MyTimelineCollectorManager extends
+ TimelineCollectorManager {
+ public MyTimelineCollectorManager() {
+ super();
+ }
+
+ @Override
+ protected CollectorNodemanagerProtocol getNMCollectorService() {
+ return mock(CollectorNodemanagerProtocol.class);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 26790f1..f974aee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -58,6 +58,11 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
</dependency>
@@ -72,6 +77,11 @@
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java
deleted file mode 100644
index 95ec9f8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/AppLevelTimelineAggregator.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-
-/**
- * Service that handles writes to the timeline service and writes them to the
- * backing storage for a given YARN application.
- *
- * App-related lifecycle management is handled by this service.
- */
-@Private
-@Unstable
-public class AppLevelTimelineAggregator extends TimelineAggregator {
- private final String applicationId;
- // TODO define key metadata such as flow metadata, user, and queue
-
- public AppLevelTimelineAggregator(String applicationId) {
- super(AppLevelTimelineAggregator.class.getName() + " - " + applicationId);
- this.applicationId = applicationId;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java
deleted file mode 100644
index 19920fd..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/PerNodeTimelineAggregatorsAuxService.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import java.nio.ByteBuffer;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.util.ShutdownHookManager;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
-import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
-import org.apache.hadoop.yarn.server.api.AuxiliaryService;
-import org.apache.hadoop.yarn.server.api.ContainerContext;
-import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
-import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
-
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * The top-level server for the per-node timeline aggregator collection. Currently
- * it is defined as an auxiliary service to accommodate running within another
- * daemon (e.g. node manager).
- */
-@Private
-@Unstable
-public class PerNodeTimelineAggregatorsAuxService extends AuxiliaryService {
- private static final Log LOG =
- LogFactory.getLog(PerNodeTimelineAggregatorsAuxService.class);
- private static final int SHUTDOWN_HOOK_PRIORITY = 30;
-
- private final TimelineAggregatorsCollection aggregatorCollection;
-
- public PerNodeTimelineAggregatorsAuxService() {
- // use the same singleton
- this(TimelineAggregatorsCollection.getInstance());
- }
-
- @VisibleForTesting PerNodeTimelineAggregatorsAuxService(
- TimelineAggregatorsCollection aggregatorCollection) {
- super("timeline_aggregator");
- this.aggregatorCollection = aggregatorCollection;
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- aggregatorCollection.init(conf);
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- aggregatorCollection.start();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- aggregatorCollection.stop();
- super.serviceStop();
- }
-
- // these methods can be used as the basis for future service methods if the
- // per-node aggregator runs separate from the node manager
- /**
- * Creates and adds an app level aggregator for the specified application id.
- * The aggregator is also initialized and started. If the service already
- * exists, no new service is created.
- *
- * @return whether it was added successfully
- */
- public boolean addApplication(ApplicationId appId) {
- AppLevelTimelineAggregator aggregator =
- new AppLevelTimelineAggregator(appId.toString());
- return (aggregatorCollection.putIfAbsent(appId, aggregator)
- == aggregator);
- }
-
- /**
- * Removes the app level aggregator for the specified application id. The
- * aggregator is also stopped as a result. If the aggregator does not exist, no
- * change is made.
- *
- * @return whether it was removed successfully
- */
- public boolean removeApplication(ApplicationId appId) {
- String appIdString = appId.toString();
- return aggregatorCollection.remove(appIdString);
- }
-
- /**
- * Creates and adds an app level aggregator for the specified application id.
- * The aggregator is also initialized and started. If the aggregator already
- * exists, no new aggregator is created.
- */
- @Override
- public void initializeContainer(ContainerInitializationContext context) {
- // intercept the event of the AM container being created and initialize the
- // app level aggregator service
- if (isApplicationMaster(context)) {
- ApplicationId appId = context.getContainerId().
- getApplicationAttemptId().getApplicationId();
- addApplication(appId);
- }
- }
-
- /**
- * Removes the app level aggregator for the specified application id. The
- * aggregator is also stopped as a result. If the aggregator does not exist, no
- * change is made.
- */
- @Override
- public void stopContainer(ContainerTerminationContext context) {
- // intercept the event of the AM container being stopped and remove the app
- // level aggregator service
- if (isApplicationMaster(context)) {
- ApplicationId appId = context.getContainerId().
- getApplicationAttemptId().getApplicationId();
- removeApplication(appId);
- }
- }
-
- private boolean isApplicationMaster(ContainerContext context) {
- // TODO this is based on a (shaky) assumption that the container id (the
- // last field of the full container id) for an AM is always 1
- // we want to make this much more reliable
- ContainerId containerId = context.getContainerId();
- return containerId.getContainerId() == 1L;
- }
-
- @VisibleForTesting
- boolean hasApplication(String appId) {
- return aggregatorCollection.containsKey(appId);
- }
-
- @Override
- public void initializeApplication(ApplicationInitializationContext context) {
- }
-
- @Override
- public void stopApplication(ApplicationTerminationContext context) {
- }
-
- @Override
- public ByteBuffer getMetaData() {
- // TODO currently it is not used; we can return a more meaningful data when
- // we connect it with an AM
- return ByteBuffer.allocate(0);
- }
-
- @VisibleForTesting
- public static PerNodeTimelineAggregatorsAuxService launchServer(String[] args) {
- Thread
- .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
- StringUtils.startupShutdownMessage(PerNodeTimelineAggregatorsAuxService.class, args,
- LOG);
- PerNodeTimelineAggregatorsAuxService auxService = null;
- try {
- auxService = new PerNodeTimelineAggregatorsAuxService();
- ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService),
- SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
- auxService.init(conf);
- auxService.start();
- } catch (Throwable t) {
- LOG.fatal("Error starting PerNodeAggregatorServer", t);
- ExitUtil.terminate(-1, "Error starting PerNodeAggregatorServer");
- }
- return auxService;
- }
-
- private static class ShutdownHook implements Runnable {
- private final PerNodeTimelineAggregatorsAuxService auxService;
-
- public ShutdownHook(PerNodeTimelineAggregatorsAuxService auxService) {
- this.auxService = auxService;
- }
-
- public void run() {
- auxService.stop();
- }
- }
-
- public static void main(String[] args) {
- launchServer(args);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java
deleted file mode 100644
index dbd0895..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregator.java
+++ /dev/null
@@ -1,122 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
-import org.apache.hadoop.util.ReflectionUtils;
-/**
- * Service that handles writes to the timeline service and writes them to the
- * backing storage.
- *
- * Classes that extend this can add their own lifecycle management or
- * customization of request handling.
- */
-@Private
-@Unstable
-public abstract class TimelineAggregator extends CompositeService {
- private static final Log LOG = LogFactory.getLog(TimelineAggregator.class);
-
- private TimelineWriter writer;
-
- public TimelineAggregator(String name) {
- super(name);
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- super.serviceInit(conf);
- writer = ReflectionUtils.newInstance(conf.getClass(
- YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
- FileSystemTimelineWriterImpl.class,
- TimelineWriter.class), conf);
- writer.init(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- writer.stop();
- }
-
- public TimelineWriter getWriter() {
- return writer;
- }
-
- /**
- * Handles entity writes. These writes are synchronous and are written to the
- * backing storage without buffering/batching. If any entity already exists,
- * it results in an update of the entity.
- *
- * This method should be reserved for selected critical entities and events.
- * For normal voluminous writes one should use the async method
- * {@link #postEntitiesAsync(TimelineEntities, UserGroupInformation)}.
- *
- * @param entities entities to post
- * @param callerUgi the caller UGI
- * @return the response that contains the result of the post.
- */
- public TimelineWriteResponse postEntities(TimelineEntities entities,
- UserGroupInformation callerUgi) throws IOException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("SUCCESS - TIMELINE V2 PROTOTYPE");
- LOG.debug("postEntities(entities=" + entities + ", callerUgi="
- + callerUgi + ")");
- }
-
- return writer.write(entities);
- }
-
- /**
- * Handles entity writes in an asynchronous manner. The method returns as soon
- * as validation is done. No promises are made on how quickly it will be
- * written to the backing storage or if it will always be written to the
- * backing storage. Multiple writes to the same entities may be batched and
- * appropriate values updated and result in fewer writes to the backing
- * storage.
- *
- * @param entities entities to post
- * @param callerUgi the caller UGI
- */
- public void postEntitiesAsync(TimelineEntities entities,
- UserGroupInformation callerUgi) {
- // TODO implement
- if (LOG.isDebugEnabled()) {
- LOG.debug("postEntitiesAsync(entities=" + entities + ", callerUgi=" +
- callerUgi + ")");
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java
deleted file mode 100644
index 7d42f94..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorWebService.java
+++ /dev/null
@@ -1,180 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import javax.servlet.ServletContext;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.*;
-import javax.ws.rs.core.Context;
-import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.webapp.ForbiddenException;
-import org.apache.hadoop.yarn.webapp.NotFoundException;
-
-import com.google.inject.Singleton;
-
-/**
- * The main per-node REST end point for timeline service writes. It is
- * essentially a container service that routes requests to the appropriate
- * per-app services.
- */
-@Private
-@Unstable
-@Singleton
-@Path("/ws/v2/timeline")
-public class TimelineAggregatorWebService {
- private static final Log LOG =
- LogFactory.getLog(TimelineAggregatorWebService.class);
-
- private @Context ServletContext context;
-
- @XmlRootElement(name = "about")
- @XmlAccessorType(XmlAccessType.NONE)
- @Public
- @Unstable
- public static class AboutInfo {
-
- private String about;
-
- public AboutInfo() {
-
- }
-
- public AboutInfo(String about) {
- this.about = about;
- }
-
- @XmlElement(name = "About")
- public String getAbout() {
- return about;
- }
-
- public void setAbout(String about) {
- this.about = about;
- }
-
- }
-
- /**
- * Return the description of the timeline web services.
- */
- @GET
- @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public AboutInfo about(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res) {
- init(res);
- return new AboutInfo("Timeline API");
- }
-
- /**
- * Accepts writes to the aggregator, and returns a response. It simply routes
- * the request to the app level aggregator. It expects an application as a
- * context.
- */
- @PUT
- @Path("/entities")
- @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
- public Response putEntities(
- @Context HttpServletRequest req,
- @Context HttpServletResponse res,
- @QueryParam("async") String async,
- @QueryParam("appid") String appId,
- TimelineEntities entities) {
- init(res);
- UserGroupInformation callerUgi = getUser(req);
- if (callerUgi == null) {
- String msg = "The owner of the posted timeline entities is not set";
- LOG.error(msg);
- throw new ForbiddenException(msg);
- }
-
- // TODO how to express async posts and handle them
- boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
-
- try {
- appId = parseApplicationId(appId);
- if (appId == null) {
- return Response.status(Response.Status.BAD_REQUEST).build();
- }
- TimelineAggregator service = getAggregatorService(req, appId);
- if (service == null) {
- LOG.error("Application not found");
- throw new NotFoundException(); // different exception?
- }
- service.postEntities(entities, callerUgi);
- return Response.ok().build();
- } catch (Exception e) {
- LOG.error("Error putting entities", e);
- throw new WebApplicationException(e,
- Response.Status.INTERNAL_SERVER_ERROR);
- }
- }
-
- private String parseApplicationId(String appId) {
- // Make sure the appId is not null and is valid
- ApplicationId appID;
- try {
- if (appId != null) {
- return ConverterUtils.toApplicationId(appId.trim()).toString();
- } else {
- return null;
- }
- } catch (Exception e) {
- return null;
- }
- }
-
- private TimelineAggregator
- getAggregatorService(HttpServletRequest req, String appIdToParse) {
- String appIdString = parseApplicationId(appIdToParse);
- final TimelineAggregatorsCollection aggregatorCollection =
- (TimelineAggregatorsCollection) context.getAttribute(
- TimelineAggregatorsCollection.AGGREGATOR_COLLECTION_ATTR_KEY);
- return aggregatorCollection.get(appIdString);
- }
-
- private void init(HttpServletResponse response) {
- response.setContentType(null);
- }
-
- private UserGroupInformation getUser(HttpServletRequest req) {
- String remoteUser = req.getRemoteUser();
- UserGroupInformation callerUgi = null;
- if (remoteUser != null) {
- callerUgi = UserGroupInformation.createRemoteUser(remoteUser);
- }
- return callerUgi;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java
deleted file mode 100644
index d6e2a18..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TimelineAggregatorsCollection.java
+++ /dev/null
@@ -1,271 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import java.io.IOException;
-import java.net.URI;
-import java.net.InetSocketAddress;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.http.lib.StaticUserWebFilter;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
-
-/**
- * Class that manages adding and removing aggregators and their lifecycle. It
- * provides thread safety access to the aggregators inside.
- *
- * It is a singleton, and instances should be obtained via
- * {@link #getInstance()}.
- */
-@Private
-@Unstable
-public class TimelineAggregatorsCollection extends CompositeService {
- private static final Log LOG =
- LogFactory.getLog(TimelineAggregatorsCollection.class);
- private static final TimelineAggregatorsCollection INSTANCE =
- new TimelineAggregatorsCollection();
-
- // access to this map is synchronized with the map itself
- private final Map<String, TimelineAggregator> aggregators =
- Collections.synchronizedMap(
- new HashMap<String, TimelineAggregator>());
-
- // REST server for this aggregator collection
- private HttpServer2 timelineRestServer;
-
- private String timelineRestServerBindAddress;
-
- private AggregatorNodemanagerProtocol nmAggregatorService;
-
- private InetSocketAddress nmAggregatorServiceAddress;
-
- static final String AGGREGATOR_COLLECTION_ATTR_KEY = "aggregator.collection";
-
- static TimelineAggregatorsCollection getInstance() {
- return INSTANCE;
- }
-
- TimelineAggregatorsCollection() {
- super(TimelineAggregatorsCollection.class.getName());
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- this.nmAggregatorServiceAddress = conf.getSocketAddr(
- YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_AGGREGATOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_PORT);
-
- }
-
- @Override
- protected void serviceStart() throws Exception {
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (timelineRestServer != null) {
- timelineRestServer.stop();
- }
- super.serviceStop();
- }
-
- /**
- * Put the aggregator into the collection if an aggregator mapped by id does
- * not exist.
- *
- * @throws YarnRuntimeException if there was any exception in initializing and
- * starting the app level service
- * @return the aggregator associated with id after the potential put.
- */
- public TimelineAggregator putIfAbsent(ApplicationId appId,
- TimelineAggregator aggregator) {
- String id = appId.toString();
- TimelineAggregator aggregatorInTable;
- boolean aggregatorIsNew = false;
- synchronized (aggregators) {
- aggregatorInTable = aggregators.get(id);
- if (aggregatorInTable == null) {
- try {
- // initialize, start, and add it to the collection so it can be
- // cleaned up when the parent shuts down
- aggregator.init(getConfig());
- aggregator.start();
- aggregators.put(id, aggregator);
- LOG.info("the aggregator for " + id + " was added");
- aggregatorInTable = aggregator;
- aggregatorIsNew = true;
- } catch (Exception e) {
- throw new YarnRuntimeException(e);
- }
- } else {
- String msg = "the aggregator for " + id + " already exists!";
- LOG.error(msg);
- }
-
- }
- // Report to NM if a new aggregator is added.
- if (aggregatorIsNew) {
- try {
- reportNewAggregatorToNM(appId);
- } catch (Exception e) {
- // throw exception here as it cannot be used if failed report to NM
- LOG.error("Failed to report a new aggregator for application: " + appId +
- " to NM Aggregator Services.");
- throw new YarnRuntimeException(e);
- }
- }
-
- return aggregatorInTable;
- }
-
- /**
- * Removes the aggregator for the specified id. The aggregator is also stopped
- * as a result. If the aggregator does not exist, no change is made.
- *
- * @return whether it was removed successfully
- */
- public boolean remove(String id) {
- synchronized (aggregators) {
- TimelineAggregator aggregator = aggregators.remove(id);
- if (aggregator == null) {
- String msg = "the aggregator for " + id + " does not exist!";
- LOG.error(msg);
- return false;
- } else {
- // stop the service to do clean up
- aggregator.stop();
- LOG.info("the aggregator service for " + id + " was removed");
- return true;
- }
- }
- }
-
- /**
- * Returns the aggregator for the specified id.
- *
- * @return the aggregator or null if it does not exist
- */
- public TimelineAggregator get(String id) {
- return aggregators.get(id);
- }
-
- /**
- * Returns whether the aggregator for the specified id exists in this
- * collection.
- */
- public boolean containsKey(String id) {
- return aggregators.containsKey(id);
- }
-
- /**
- * Launch the REST web server for this aggregator collection
- */
- private void startWebApp() {
- Configuration conf = getConfig();
- // use the same ports as the old ATS for now; we could create new properties
- // for the new timeline service if needed
- String bindAddress = WebAppUtils.getWebAppBindURL(conf,
- YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
- WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
- this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
- NetUtils.createSocketAddr(bindAddress));
- LOG.info("Instantiating the per-node aggregator webapp at " +
- timelineRestServerBindAddress);
- try {
- Configuration confForInfoServer = new Configuration(conf);
- confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
- HttpServer2.Builder builder = new HttpServer2.Builder()
- .setName("timeline")
- .setConf(conf)
- .addEndpoint(URI.create("http://" + bindAddress));
- timelineRestServer = builder.build();
- // TODO: replace this by an authentication filter in future.
- HashMap<String, String> options = new HashMap<>();
- String username = conf.get(HADOOP_HTTP_STATIC_USER,
- DEFAULT_HADOOP_HTTP_STATIC_USER);
- options.put(HADOOP_HTTP_STATIC_USER, username);
- HttpServer2.defineFilter(timelineRestServer.getWebAppContext(),
- "static_user_filter_timeline",
- StaticUserWebFilter.StaticUserFilter.class.getName(),
- options, new String[] {"/*"});
-
- timelineRestServer.addJerseyResourcePackage(
- TimelineAggregatorWebService.class.getPackage().getName() + ";"
- + GenericExceptionHandler.class.getPackage().getName() + ";"
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
- "/*");
- timelineRestServer.setAttribute(AGGREGATOR_COLLECTION_ATTR_KEY,
- TimelineAggregatorsCollection.getInstance());
- timelineRestServer.start();
- } catch (Exception e) {
- String msg = "The per-node aggregator webapp failed to start.";
- LOG.error(msg, e);
- throw new YarnRuntimeException(msg, e);
- }
- }
-
- private void reportNewAggregatorToNM(ApplicationId appId)
- throws YarnException, IOException {
- this.nmAggregatorService = getNMAggregatorService();
- ReportNewAggregatorsInfoRequest request =
- ReportNewAggregatorsInfoRequest.newInstance(appId,
- this.timelineRestServerBindAddress);
- LOG.info("Report a new aggregator for application: " + appId +
- " to NM Aggregator Services.");
- nmAggregatorService.reportNewAggregatorInfo(request);
- }
-
- // protected for test
- protected AggregatorNodemanagerProtocol getNMAggregatorService(){
- Configuration conf = getConfig();
- final YarnRPC rpc = YarnRPC.create(conf);
-
- // TODO Security settings.
- return (AggregatorNodemanagerProtocol) rpc.getProxy(
- AggregatorNodemanagerProtocol.class,
- nmAggregatorServiceAddress, conf);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
new file mode 100644
index 0000000..7d59876
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * Service that handles writes to the timeline service and writes them to the
+ * backing storage for a given YARN application.
+ *
+ * App-related lifecycle management is handled by this service.
+ */
+@Private
+@Unstable
+public class AppLevelTimelineCollector extends TimelineCollector {
+ private final String applicationId;
+ // TODO define key metadata such as flow metadata, user, and queue
+
+ public AppLevelTimelineCollector(String applicationId) {
+ super(AppLevelTimelineCollector.class.getName() + " - " + applicationId);
+ this.applicationId = applicationId;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+}
[02/50] [abbrv] hadoop git commit: YARN-3040. Make putEntities
operation be aware of the app's context. Contributed by Zhijie Shen
Posted by sj...@apache.org.
YARN-3040. Make putEntities operation be aware of the app's context. Contributed by Zhijie Shen
(cherry picked from commit db2f0238915d6e1a5b85c463426b5e072bd4698d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8b5ab64
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8b5ab64
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8b5ab64
Branch: refs/heads/YARN-2928
Commit: e8b5ab64d4ecee98d3463826c50ce18dfa7eea43
Parents: a798523
Author: Junping Du <ju...@apache.org>
Authored: Thu Mar 26 09:59:32 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:38:43 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 1 +
.../applications/distributedshell/Client.java | 27 +++-
.../distributedshell/TestDistributedShell.java | 125 +++++++++-------
.../yarn/util/timeline/TimelineUtils.java | 16 +++
.../api/CollectorNodemanagerProtocol.java | 16 +++
...ollectorNodemanagerProtocolPBClientImpl.java | 20 +++
...llectorNodemanagerProtocolPBServiceImpl.java | 21 +++
.../GetTimelineCollectorContextRequest.java | 37 +++++
.../GetTimelineCollectorContextResponse.java | 46 ++++++
...etTimelineCollectorContextRequestPBImpl.java | 127 +++++++++++++++++
...tTimelineCollectorContextResponsePBImpl.java | 141 +++++++++++++++++++
.../proto/collectornodemanager_protocol.proto | 1 +
.../yarn_server_common_service_protos.proto | 9 ++
.../java/org/apache/hadoop/yarn/TestRPC.java | 39 +++++
.../collectormanager/NMCollectorService.java | 18 ++-
.../containermanager/ContainerManagerImpl.java | 14 +-
.../application/Application.java | 4 +
.../application/ApplicationImpl.java | 17 ++-
.../application/TestApplication.java | 3 +-
.../yarn/server/nodemanager/webapp/MockApp.java | 10 ++
.../nodemanager/webapp/TestNMWebServices.java | 2 +-
.../resourcemanager/amlauncher/AMLauncher.java | 23 ++-
.../timelineservice/RMTimelineCollector.java | 7 +
.../TestTimelineServiceClientIntegration.java | 19 ++-
.../collector/AppLevelTimelineCollector.java | 33 ++++-
.../PerNodeTimelineCollectorsAuxService.java | 2 +-
.../collector/TimelineCollector.java | 19 ++-
.../collector/TimelineCollectorContext.java | 81 +++++++++++
.../collector/TimelineCollectorManager.java | 32 ++++-
.../collector/TimelineCollectorWebService.java | 2 +-
.../storage/FileSystemTimelineWriterImpl.java | 69 +++++----
.../timelineservice/storage/TimelineWriter.java | 9 +-
...TestPerNodeTimelineCollectorsAuxService.java | 43 ++++--
.../collector/TestTimelineCollectorManager.java | 41 +++++-
.../TestFileSystemTimelineWriterImpl.java | 22 ++-
36 files changed, 956 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 82ed275..ed5dc88 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -41,6 +41,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3034. Implement RM starting its timeline collector. (Naganarasimha G R
via junping_du)
+ YARN-3040. Make putEntities operation be aware of the app's context. (Zhijie Shen
+ via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e72c1c0..51f2b2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -124,6 +124,7 @@ public class YarnConfiguration extends Configuration {
public static final String RM_PREFIX = "yarn.resourcemanager.";
public static final String RM_CLUSTER_ID = RM_PREFIX + "cluster-id";
+ public static final String DEFAULT_RM_CLUSTER_ID = "yarn_cluster";
public static final String RM_HOSTNAME = RM_PREFIX + "hostname";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 163f8f0..db69490 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -22,8 +22,10 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.Vector;
import org.apache.commons.cli.CommandLine;
@@ -183,6 +185,9 @@ public class Client {
// Timeline domain writer access control
private String modifyACLs = null;
+ private String flowId = null;
+ private String flowRunId = null;
+
// Command line options
private Options opts;
@@ -256,7 +261,8 @@ public class Client {
opts.addOption("shell_args", true, "Command line args for the shell script." +
"Multiple args can be separated by empty space.");
opts.getOption("shell_args").setArgs(Option.UNLIMITED_VALUES);
- opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs");
+ opts.addOption("shell_env", true,
+ "Environment for shell script. Specified as env_key=env_val pairs");
opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers");
opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command");
opts.addOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command");
@@ -283,6 +289,10 @@ public class Client {
+ "modify the timeline entities in the given domain");
opts.addOption("create", false, "Flag to indicate whether to create the "
+ "domain specified with -domain.");
+ opts.addOption("flow", true, "ID of the flow which the distributed shell "
+ + "app belongs to");
+ opts.addOption("flow_run", true, "ID of the flowrun which the distributed "
+ + "shell app belongs to");
opts.addOption("help", false, "Print usage");
opts.addOption("node_label_expression", true,
"Node label expression to determine the nodes"
@@ -442,6 +452,12 @@ public class Client {
}
}
+ if (cliParser.hasOption("flow")) {
+ flowId = cliParser.getOptionValue("flow");
+ }
+ if (cliParser.hasOption("flow_run")) {
+ flowRunId = cliParser.getOptionValue("flow_run");
+ }
return true;
}
@@ -533,6 +549,15 @@ public class Client {
.setAttemptFailuresValidityInterval(attemptFailuresValidityInterval);
}
+ Set<String> tags = new HashSet<String>();
+ if (flowId != null) {
+ tags.add(TimelineUtils.generateFlowIdTag(flowId));
+ }
+ if (flowRunId != null) {
+ tags.add(TimelineUtils.generateFlowRunIdTag(flowRunId));
+ }
+ appContext.setApplicationTags(tags);
+
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of the local resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 0af050c..1de3b68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -33,12 +33,14 @@ import java.util.Arrays;
import java.util.List;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -52,6 +54,7 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -161,20 +164,26 @@ public class TestDistributedShell {
@Test(timeout=90000)
public void testDSShellWithDomain() throws Exception {
- testDSShell(true, "v1");
+ testDSShell(true, "v1", true);
}
@Test(timeout=90000)
public void testDSShellWithoutDomain() throws Exception {
- testDSShell(false, "v1");
+ testDSShell(false, "v1", true);
}
@Test(timeout=90000)
- public void testDSShellWithoutDomainV2() throws Exception {
- testDSShell(false, "v2");
+ public void testDSShellWithoutDomainV2DefaultFlow() throws Exception {
+ testDSShell(false, "v2", true);
}
- public void testDSShell(boolean haveDomain, String timelineVersion)
+ @Test(timeout=90000)
+ public void testDSShellWithoutDomainV2CustomizedFlow() throws Exception {
+ testDSShell(false, "v2", false);
+ }
+
+ public void testDSShell(boolean haveDomain, String timelineVersion,
+ boolean defaultFlow)
throws Exception {
String[] args = {
"--jar",
@@ -212,6 +221,15 @@ public class TestDistributedShell {
};
isTestingTimelineV2 = true;
args = mergeArgs(args, timelineArgs);
+ if (!defaultFlow) {
+ String[] flowArgs = {
+ "--flow",
+ "test_flow_id",
+ "--flow_run",
+ "12345678"
+ };
+ args = mergeArgs(args, flowArgs);
+ }
LOG.info("Setup: Using timeline v2!");
}
@@ -271,7 +289,7 @@ public class TestDistributedShell {
if (!isTestingTimelineV2) {
checkTimelineV1(haveDomain);
} else {
- checkTimelineV2(haveDomain, appId);
+ checkTimelineV2(haveDomain, appId, defaultFlow);
}
}
@@ -320,53 +338,58 @@ public class TestDistributedShell {
}
}
- private void checkTimelineV2(boolean haveDomain, ApplicationId appId) {
- // For PoC check in /tmp/ YARN-3264
- String tmpRoot = FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
+ private void checkTimelineV2(
+ boolean haveDomain, ApplicationId appId, boolean defaultFlow)
+ throws Exception {
+ // For PoC check in /tmp/timeline_service_data YARN-3264
+ String tmpRoot =
+ FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
+ + "/entities/";
File tmpRootFolder = new File(tmpRoot);
- Assert.assertTrue(tmpRootFolder.isDirectory());
-
- // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
- String outputDirApp = tmpRoot + "/DS_APP_ATTEMPT/";
-
- File entityFolder = new File(outputDirApp);
- Assert.assertTrue(entityFolder.isDirectory());
-
- // there will be at least one attempt, look for that file
- String appTimestampFileName = "appattempt_" + appId.getClusterTimestamp()
- + "_000" + appId.getId() + "_000001"
- + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
- String appAttemptFileName = outputDirApp + appTimestampFileName;
- File appAttemptFile = new File(appAttemptFileName);
- Assert.assertTrue(appAttemptFile.exists());
-
- String outputDirContainer = tmpRoot + "/DS_CONTAINER/";
- File containerFolder = new File(outputDirContainer);
- Assert.assertTrue(containerFolder.isDirectory());
-
- String containerTimestampFileName = "container_"
- + appId.getClusterTimestamp() + "_000" + appId.getId()
- + "_01_000002.thist";
- String containerFileName = outputDirContainer + containerTimestampFileName;
- File containerFile = new File(containerFileName);
- Assert.assertTrue(containerFile.exists());
- String appTimeStamp = appId.getClusterTimestamp() + "_" + appId.getId()
- + "_";
- deleteAppFiles(new File(outputDirApp), appTimeStamp);
- deleteAppFiles(new File(outputDirContainer), appTimeStamp);
- tmpRootFolder.delete();
- }
-
- private void deleteAppFiles(File rootDir, String appTimeStamp) {
- boolean deleted = false;
- File[] listOfFiles = rootDir.listFiles();
- for (File f1 : listOfFiles) {
- // list all attempts for this app and delete them
- if (f1.getName().contains(appTimeStamp)){
- deleted = f1.delete();
- Assert.assertTrue(deleted);
- }
+ try {
+ Assert.assertTrue(tmpRootFolder.isDirectory());
+
+ // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
+ String outputDirApp = tmpRoot +
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
+ UserGroupInformation.getCurrentUser().getShortUserName() +
+ (defaultFlow ? "/" +
+ TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+ "/0/" : "/test_flow_id/12345678/") +
+ appId.toString() + "/DS_APP_ATTEMPT/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // there will be at least one attempt, look for that file
+ String appTimestampFileName = "appattempt_" + appId.getClusterTimestamp()
+ + "_000" + appId.getId() + "_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ String appAttemptFileName = outputDirApp + appTimestampFileName;
+ File appAttemptFile = new File(appAttemptFileName);
+ Assert.assertTrue(appAttemptFile.exists());
+
+ String outputDirContainer = tmpRoot +
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
+ UserGroupInformation.getCurrentUser().getShortUserName() +
+ (defaultFlow ? "/" +
+ TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+ "/0/" : "/test_flow_id/12345678/") +
+ appId.toString() + "/DS_CONTAINER/";
+ File containerFolder = new File(outputDirContainer);
+ Assert.assertTrue(containerFolder.isDirectory());
+
+ String containerTimestampFileName = "container_"
+ + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_01_000002.thist";
+ String containerFileName = outputDirContainer + containerTimestampFileName;
+ File containerFile = new File(containerFileName);
+ Assert.assertTrue(containerFile.exists());
+ String appTimeStamp = appId.getClusterTimestamp() + "_" + appId.getId()
+ + "_";
+ } finally {
+ FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index 4f838e6..ce14562 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -43,6 +44,9 @@ import org.codehaus.jackson.map.ObjectMapper;
@Evolving
public class TimelineUtils {
+ public static final String FLOW_ID_TAG_PREFIX = "TIMELINE_FLOW_ID_TAG";
+ public static final String FLOW_RUN_ID_TAG_PREFIX = "TIMELINE_FLOW_RUN_ID_TAG";
+
private static ObjectMapper mapper;
static {
@@ -119,4 +123,16 @@ public class TimelineUtils {
getTimelineTokenServiceAddress(conf);
return SecurityUtil.buildTokenService(timelineServiceAddr);
}
+
+ public static String generateDefaultFlowIdBasedOnAppId(ApplicationId appId) {
+ return "flow_" + appId.getClusterTimestamp() + "_" + appId.getId();
+ }
+
+ public static String generateFlowIdTag(String flowId) {
+ return FLOW_ID_TAG_PREFIX + ":" + flowId;
+ }
+
+ public static String generateFlowRunIdTag(String flowRunId) {
+ return FLOW_RUN_ID_TAG_PREFIX + ":" + flowRunId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
index 26c121a..d23c04a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
@@ -54,4 +56,18 @@ public interface CollectorNodemanagerProtocol {
ReportNewCollectorInfoRequest request)
throws YarnException, IOException;
+ /**
+ * <p>
+ * The collector needs to get the context information including user, flow
+ * and flow run ID to associate with every incoming put-entity requests.
+ * </p>
+ * @param request the request of getting the aggregator context information of
+ * the given application
+ * @return
+ * @throws YarnException
+ * @throws IOException
+ */
+ GetTimelineCollectorContextResponse getTimelineCollectorContext(
+ GetTimelineCollectorContextRequest request)
+ throws YarnException, IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
index 276a540..b9e17f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
@@ -30,11 +30,16 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetTimelineCollectorContextRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetTimelineCollectorContextResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl;
@@ -85,6 +90,21 @@ public class CollectorNodemanagerProtocolPBClientImpl implements
}
@Override
+ public GetTimelineCollectorContextResponse getTimelineCollectorContext(
+ GetTimelineCollectorContextRequest request)
+ throws YarnException, IOException {
+ GetTimelineCollectorContextRequestProto requestProto =
+ ((GetTimelineCollectorContextRequestPBImpl) request).getProto();
+ try {
+ return new GetTimelineCollectorContextResponsePBImpl(
+ proxy.getTimelineCollectorContext(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
+ @Override
public void close() {
if (this.proxy != null) {
RPC.stopProxy(this.proxy);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
index 3f42732..21fb270 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
@@ -20,11 +20,16 @@ package org.apache.hadoop.yarn.server.api.impl.pb.service;
import java.io.IOException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetTimelineCollectorContextRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.GetTimelineCollectorContextResponsePBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl;
@@ -56,4 +61,20 @@ public class CollectorNodemanagerProtocolPBServiceImpl implements
}
}
+ @Override
+ public GetTimelineCollectorContextResponseProto getTimelineCollectorContext(
+ RpcController controller,
+ GetTimelineCollectorContextRequestProto proto) throws ServiceException {
+ GetTimelineCollectorContextRequestPBImpl request =
+ new GetTimelineCollectorContextRequestPBImpl(proto);
+ try {
+ GetTimelineCollectorContextResponse response =
+ real.getTimelineCollectorContext(request);
+ return ((GetTimelineCollectorContextResponsePBImpl)response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextRequest.java
new file mode 100644
index 0000000..604a40b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextRequest.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+public abstract class GetTimelineCollectorContextRequest {
+
+ public static GetTimelineCollectorContextRequest newInstance(
+ ApplicationId appId) {
+ GetTimelineCollectorContextRequest request =
+ Records.newRecord(GetTimelineCollectorContextRequest.class);
+ request.setApplicationId(appId);
+ return request;
+ }
+
+ public abstract ApplicationId getApplicationId();
+
+ public abstract void setApplicationId(ApplicationId appId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
new file mode 100644
index 0000000..1558e2f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+
+import org.apache.hadoop.yarn.util.Records;
+
+public abstract class GetTimelineCollectorContextResponse {
+
+ public static GetTimelineCollectorContextResponse newInstance(
+ String userId, String flowId, String flowRunId) {
+ GetTimelineCollectorContextResponse response =
+ Records.newRecord(GetTimelineCollectorContextResponse.class);
+ response.setUserId(userId);
+ response.setFlowId(flowId);
+ response.setFlowRunId(flowRunId);
+ return response;
+ }
+
+ public abstract String getUserId();
+
+ public abstract void setUserId(String userId);
+
+ public abstract String getFlowId();
+
+ public abstract void setFlowId(String flowId);
+
+ public abstract String getFlowRunId();
+
+ public abstract void setFlowRunId(String flowRunId);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextRequestPBImpl.java
new file mode 100644
index 0000000..b53b55b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextRequestPBImpl.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+
+import com.google.protobuf.TextFormat;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextRequestProto;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+
+public class GetTimelineCollectorContextRequestPBImpl extends
+ GetTimelineCollectorContextRequest {
+
+ GetTimelineCollectorContextRequestProto
+ proto = GetTimelineCollectorContextRequestProto.getDefaultInstance();
+ GetTimelineCollectorContextRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ApplicationId appId = null;
+
+ public GetTimelineCollectorContextRequestPBImpl() {
+ builder = GetTimelineCollectorContextRequestProto.newBuilder();
+ }
+
+ public GetTimelineCollectorContextRequestPBImpl(
+ GetTimelineCollectorContextRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetTimelineCollectorContextRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToBuilder() {
+ if (appId != null) {
+ builder.setAppId(convertToProtoFormat(this.appId));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetTimelineCollectorContextRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ if (this.appId != null) {
+ return this.appId;
+ }
+
+ GetTimelineCollectorContextRequestProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasAppId()) {
+ return null;
+ }
+
+ this.appId = convertFromProtoFormat(p.getAppId());
+ return this.appId;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId appId) {
+ maybeInitBuilder();
+ if (appId == null)
+ builder.clearAppId();
+ this.appId = appId;
+ }
+
+ private ApplicationIdPBImpl convertFromProtoFormat(YarnProtos.ApplicationIdProto p) {
+ return new ApplicationIdPBImpl(p);
+ }
+
+ private YarnProtos.ApplicationIdProto convertToProtoFormat(ApplicationId t) {
+ return ((ApplicationIdPBImpl)t).getProto();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
new file mode 100644
index 0000000..6dc1f77
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+
+import com.google.protobuf.TextFormat;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.GetTimelineCollectorContextResponseProto;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
+
+public class GetTimelineCollectorContextResponsePBImpl extends
+ GetTimelineCollectorContextResponse {
+
+ GetTimelineCollectorContextResponseProto proto =
+ GetTimelineCollectorContextResponseProto.getDefaultInstance();
+ GetTimelineCollectorContextResponseProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public GetTimelineCollectorContextResponsePBImpl() {
+ builder = GetTimelineCollectorContextResponseProto.newBuilder();
+ }
+
+ public GetTimelineCollectorContextResponsePBImpl(
+ GetTimelineCollectorContextResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public GetTimelineCollectorContextResponseProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = GetTimelineCollectorContextResponseProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public String getUserId() {
+ GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasUserId()) {
+ return null;
+ }
+ return p.getUserId();
+ }
+
+ @Override
+ public void setUserId(String userId) {
+ maybeInitBuilder();
+ if (userId == null) {
+ builder.clearUserId();
+ return;
+ }
+ builder.setUserId(userId);
+ }
+
+ @Override
+ public String getFlowId() {
+ GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasFlowId()) {
+ return null;
+ }
+ return p.getFlowId();
+ }
+
+ @Override
+ public void setFlowId(String flowId) {
+ maybeInitBuilder();
+ if (flowId == null) {
+ builder.clearFlowId();
+ return;
+ }
+ builder.setFlowId(flowId);
+ }
+
+ @Override
+ public String getFlowRunId() {
+ GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasFlowRunId()) {
+ return null;
+ }
+ return p.getFlowRunId();
+ }
+
+ @Override
+ public void setFlowRunId(String flowRunId) {
+ maybeInitBuilder();
+ if (flowRunId == null) {
+ builder.clearFlowRunId();
+ return;
+ }
+ builder.setFlowRunId(flowRunId);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
index 654a9f2..8665274 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
@@ -26,4 +26,5 @@ import "yarn_server_common_service_protos.proto";
service CollectorNodemanagerProtocolService {
rpc reportNewCollectorInfo (ReportNewCollectorInfoRequestProto) returns (ReportNewCollectorInfoResponseProto);
+ rpc getTimelineCollectorContext (GetTimelineCollectorContextRequestProto) returns (GetTimelineCollectorContextResponseProto);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index d358b83..2234752 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -109,6 +109,15 @@ message ReportNewCollectorInfoRequestProto {
message ReportNewCollectorInfoResponseProto {
}
+message GetTimelineCollectorContextRequestProto {
+ optional ApplicationIdProto appId = 1;
+}
+
+message GetTimelineCollectorContextResponseProto {
+ optional string user_id = 1;
+ optional string flow_id = 2;
+ optional string flow_run_id = 3;
+}
message NMContainerStatusProto {
optional ContainerIdProto container_id = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
index cfc3dc6..3c9f57b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
@@ -60,6 +60,8 @@ import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
@@ -166,6 +168,31 @@ public class TestRPC {
Assert.assertTrue(e.getMessage().contains(ILLEGAL_NUMBER_MESSAGE));
}
+ // Verify request with a valid app ID
+ try {
+ GetTimelineCollectorContextRequest request =
+ GetTimelineCollectorContextRequest.newInstance(
+ ApplicationId.newInstance(0, 1));
+ GetTimelineCollectorContextResponse response =
+ proxy.getTimelineCollectorContext(request);
+ Assert.assertEquals("test_user_id", response.getUserId());
+ Assert.assertEquals("test_flow_id", response.getFlowId());
+ Assert.assertEquals("test_flow_run_id", response.getFlowRunId());
+ } catch (YarnException | IOException e) {
+ Assert.fail("RPC call failured is not expected here.");
+ }
+
+ // Verify request with an invalid app ID
+ try {
+ GetTimelineCollectorContextRequest request =
+ GetTimelineCollectorContextRequest.newInstance(
+ ApplicationId.newInstance(0, 2));
+ proxy.getTimelineCollectorContext(request);
+ Assert.fail("RPC call failured is expected here.");
+ } catch (YarnException | IOException e) {
+ Assert.assertTrue(e instanceof YarnException);
+ Assert.assertTrue(e.getMessage().contains("The application is not found."));
+ }
server.stop();
}
@@ -340,6 +367,18 @@ public class TestRPC {
recordFactory.newRecordInstance(ReportNewCollectorInfoResponse.class);
return response;
}
+
+ @Override
+ public GetTimelineCollectorContextResponse getTimelineCollectorContext(
+ GetTimelineCollectorContextRequest request)
+ throws YarnException, IOException {
+ if (request.getApplicationId().getId() == 1) {
+ return GetTimelineCollectorContextResponse.newInstance(
+ "test_user_id", "test_flow_id", "test_flow_run_id");
+ } else {
+ throw new YarnException("The application is not found.");
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index 009fa63..6ccea84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -30,13 +30,17 @@ import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
public class NMCollectorService extends CompositeService implements
CollectorNodemanagerProtocol {
@@ -93,7 +97,7 @@ public class NMCollectorService extends CompositeService implements
@Override
public ReportNewCollectorInfoResponse reportNewCollectorInfo(
- ReportNewCollectorInfoRequest request) throws IOException {
+ ReportNewCollectorInfoRequest request) throws YarnException, IOException {
List<AppCollectorsMap> newCollectorsList = request.getAppCollectorsList();
if (newCollectorsList != null && !newCollectorsList.isEmpty()) {
Map<ApplicationId, String> newCollectorsMap =
@@ -107,4 +111,16 @@ public class NMCollectorService extends CompositeService implements
return ReportNewCollectorInfoResponse.newInstance();
}
+ @Override
+ public GetTimelineCollectorContextResponse getTimelineCollectorContext(
+ GetTimelineCollectorContextRequest request)
+ throws YarnException, IOException {
+ Application app = context.getApplications().get(request.getApplicationId());
+ if (app == null) {
+ throw new YarnException("Application " + request.getApplicationId() +
+ " doesn't exist on NM.");
+ }
+ return GetTimelineCollectorContextResponse.newInstance(
+ app.getUser(), app.getFlowId(), app.getFlowRunId());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 494fa8f..8a60dcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -139,6 +139,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
public class ContainerManagerImpl extends CompositeService implements
ServiceStateChangeListener, ContainerManagementProtocol,
@@ -297,8 +298,9 @@ public class ContainerManagerImpl extends CompositeService implements
}
LOG.info("Recovering application " + appId);
- ApplicationImpl app = new ApplicationImpl(dispatcher, p.getUser(), appId,
- creds, context);
+ //TODO: Recover flow and flow run ID
+ ApplicationImpl app = new ApplicationImpl(
+ dispatcher, p.getUser(), null, null, appId, creds, context);
context.getApplications().put(appId, app);
app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext));
}
@@ -868,8 +870,12 @@ public class ContainerManagerImpl extends CompositeService implements
try {
if (!serviceStopped) {
// Create the application
- Application application =
- new ApplicationImpl(dispatcher, user, applicationID, credentials, context);
+ String flowId = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_ID_TAG_PREFIX);
+ String flowRunId = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
+ Application application = new ApplicationImpl(
+ dispatcher, user, flowId, flowRunId, applicationID, credentials, context);
if (null == context.getApplications().putIfAbsent(applicationID,
application)) {
LOG.info("Creating a new application reference for app " + applicationID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
index b1571e9..decd17d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
@@ -35,4 +35,8 @@ public interface Application extends EventHandler<ApplicationEvent> {
ApplicationState getApplicationState();
+ String getFlowId();
+
+ String getFlowRunId();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index cc82401..67604d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -64,6 +64,8 @@ public class ApplicationImpl implements Application {
final Dispatcher dispatcher;
final String user;
+ final String flowId;
+ final String flowRunId;
final ApplicationId appId;
final Credentials credentials;
Map<ApplicationAccessType, String> applicationACLs;
@@ -79,10 +81,13 @@ public class ApplicationImpl implements Application {
Map<ContainerId, Container> containers =
new HashMap<ContainerId, Container>();
- public ApplicationImpl(Dispatcher dispatcher, String user, ApplicationId appId,
- Credentials credentials, Context context) {
+ public ApplicationImpl(Dispatcher dispatcher, String user, String flowId,
+ String flowRunId, ApplicationId appId, Credentials credentials,
+ Context context) {
this.dispatcher = dispatcher;
this.user = user;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = context.getApplicationACLsManager();
@@ -487,4 +492,12 @@ public class ApplicationImpl implements Application {
this.readLock.unlock();
}
}
+
+ public String getFlowId() {
+ return flowId;
+ }
+
+ public String getFlowRunId() {
+ return flowRunId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 370a207..5303df5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -530,7 +530,8 @@ public class TestApplication {
this.user = user;
this.appId = BuilderUtils.newApplicationId(timestamp, id);
- app = new ApplicationImpl(dispatcher, this.user, appId, null, context);
+ app = new ApplicationImpl(
+ dispatcher, this.user, null, null, appId, null, context);
containers = new ArrayList<Container>();
for (int i = 0; i < numContainers; i++) {
Container container = createMockedContainer(this.appId, i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
index 4e13010..35b95ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
@@ -39,6 +39,9 @@ public class MockApp implements Application {
Map<ContainerId, Container> containers = new HashMap<ContainerId, Container>();
ApplicationState appState;
Application app;
+ String flowId;
+ String flowRunId;
+
public MockApp(int uniqId) {
this("mockUser", 1234, uniqId);
@@ -77,4 +80,11 @@ public class MockApp implements Application {
public void handle(ApplicationEvent event) {}
+ public String getFlowId() {
+ return flowId;
+ }
+
+ public String getFlowRunId() {
+ return flowRunId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 1f5590c..2ac9c22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -327,7 +327,7 @@ public class TestNMWebServices extends JerseyTestBase {
final String filename = "logfile1";
final String logMessage = "log message\n";
nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
- appId, null, nmContext));
+ null, null, appId, null, nmContext));
MockContainer container = new MockContainer(appAttemptId,
new AsyncDispatcher(), new Configuration(), "user", appId, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 713e75f..5c0b02b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -35,8 +35,8 @@ import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
+import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.ApplicationConstants;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptLaunchFailedEvent;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -218,6 +219,26 @@ public class AMLauncher implements Runnable {
.get(applicationId)
.getSubmitTime()));
+ // Set flow context info
+ for (String tag :
+ rmContext.getRMApps().get(applicationId).getApplicationTags()) {
+ if (tag.startsWith(TimelineUtils.FLOW_ID_TAG_PREFIX + ":") ||
+ tag.startsWith(TimelineUtils.FLOW_ID_TAG_PREFIX.toLowerCase() + ":")) {
+ String value = tag.substring(
+ TimelineUtils.FLOW_ID_TAG_PREFIX.length() + 1);
+ if (!value.isEmpty()) {
+ environment.put(TimelineUtils.FLOW_ID_TAG_PREFIX, value);
+ }
+ }
+ if (tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX + ":") ||
+ tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) {
+ String value = tag.substring(
+ TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1);
+ if (!value.isEmpty()) {
+ environment.put(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, value);
+ }
+ }
+ }
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = container.getTokens();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
index 22743d6..4ea7a03 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEvent;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEventType;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
/**
* This class is responsible for posting application and appattempt lifecycle
@@ -87,6 +88,12 @@ public class RMTimelineCollector extends TimelineCollector {
LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
}
}
+
+ @Override
+ protected TimelineCollectorContext getTimelineEntityContext() {
+ // TODO address in YARN-3390.
+ return null;
+ }
/**
* EventHandler implementation which forward events to SystemMetricsPublisher.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index fab131c..c8b9625 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -20,20 +20,27 @@ package org.apache.hadoop.yarn.server.timelineservice;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
+import java.io.IOException;
+
public class TestTimelineServiceClientIntegration {
private static TimelineCollectorManager collectorManager;
private static PerNodeTimelineCollectorsAuxService auxService;
@@ -86,7 +93,17 @@ public class TestTimelineServiceClientIntegration {
@Override
protected CollectorNodemanagerProtocol getNMCollectorService() {
- return mock(CollectorNodemanagerProtocol.class);
+ CollectorNodemanagerProtocol protocol =
+ mock(CollectorNodemanagerProtocol.class);
+ try {
+ GetTimelineCollectorContextResponse response =
+ GetTimelineCollectorContextResponse.newInstance(null, null, null);
+ when(protocol.getTimelineCollectorContext(any(
+ GetTimelineCollectorContextRequest.class))).thenReturn(response);
+ } catch (YarnException | IOException e) {
+ fail();
+ }
+ return protocol;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index 7d59876..60ddde5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -18,9 +18,14 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
/**
* Service that handles writes to the timeline service and writes them to the
@@ -31,16 +36,29 @@ import org.apache.hadoop.conf.Configuration;
@Private
@Unstable
public class AppLevelTimelineCollector extends TimelineCollector {
- private final String applicationId;
- // TODO define key metadata such as flow metadata, user, and queue
+ private final ApplicationId appId;
+ private final TimelineCollectorContext context;
- public AppLevelTimelineCollector(String applicationId) {
- super(AppLevelTimelineCollector.class.getName() + " - " + applicationId);
- this.applicationId = applicationId;
+ public AppLevelTimelineCollector(ApplicationId appId) {
+ super(AppLevelTimelineCollector.class.getName() + " - " + appId.toString());
+ Preconditions.checkNotNull(appId, "AppId shouldn't be null");
+ this.appId = appId;
+ context = new TimelineCollectorContext();
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
+ context.setClusterId(conf.get(YarnConfiguration.RM_CLUSTER_ID,
+ YarnConfiguration.DEFAULT_RM_CLUSTER_ID));
+ // Set the default values, which will be updated with an RPC call to get the
+ // context info from NM.
+ // Current user usually is not the app user, but keep this field non-null
+ context.setUserId(UserGroupInformation.getCurrentUser().getShortUserName());
+ // Use app ID to generate a default flow ID for orphan app
+ context.setFlowId(TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId));
+ // Set the flow run ID to 0 if it's an orphan app
+ context.setFlowRunId("0");
+ context.setAppId(appId.toString());
super.serviceInit(conf);
}
@@ -54,4 +72,9 @@ public class AppLevelTimelineCollector extends TimelineCollector {
super.serviceStop();
}
+ @Override
+ protected TimelineCollectorContext getTimelineEntityContext() {
+ return context;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 59ecef1..2017d01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -95,7 +95,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
*/
public boolean addApplication(ApplicationId appId) {
AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId.toString());
+ new AppLevelTimelineCollector(appId);
return (collectorManager.putIfAbsent(appId, collector)
== collector);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 6e20e69..677feb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
+
/**
* Service that handles writes to the timeline service and writes them to the
* backing storage.
@@ -83,21 +84,24 @@ public abstract class TimelineCollector extends CompositeService {
*
* This method should be reserved for selected critical entities and events.
* For normal voluminous writes one should use the async method
- * {@link #postEntitiesAsync(TimelineEntities, UserGroupInformation)}.
+ * {@link #putEntitiesAsync(TimelineEntities, UserGroupInformation)}.
*
* @param entities entities to post
* @param callerUgi the caller UGI
* @return the response that contains the result of the post.
*/
- public TimelineWriteResponse postEntities(TimelineEntities entities,
+ public TimelineWriteResponse putEntities(TimelineEntities entities,
UserGroupInformation callerUgi) throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("SUCCESS - TIMELINE V2 PROTOTYPE");
- LOG.debug("postEntities(entities=" + entities + ", callerUgi="
+ LOG.debug("putEntities(entities=" + entities + ", callerUgi="
+ callerUgi + ")");
}
- return writer.write(entities);
+ TimelineCollectorContext context = getTimelineEntityContext();
+ return writer.write(context.getClusterId(), context.getUserId(),
+ context.getFlowId(), context.getFlowRunId(), context.getAppId(),
+ entities);
}
/**
@@ -111,12 +115,15 @@ public abstract class TimelineCollector extends CompositeService {
* @param entities entities to post
* @param callerUgi the caller UGI
*/
- public void postEntitiesAsync(TimelineEntities entities,
+ public void putEntitiesAsync(TimelineEntities entities,
UserGroupInformation callerUgi) {
// TODO implement
if (LOG.isDebugEnabled()) {
- LOG.debug("postEntitiesAsync(entities=" + entities + ", callerUgi=" +
+ LOG.debug("putEntitiesAsync(entities=" + entities + ", callerUgi=" +
callerUgi + ")");
}
}
+
+ protected abstract TimelineCollectorContext getTimelineEntityContext();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
new file mode 100644
index 0000000..c1a10a6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+public class TimelineCollectorContext {
+
+ private String clusterId;
+ private String userId;
+ private String flowId;
+ private String flowRunId;
+ private String appId;
+
+ public TimelineCollectorContext() {
+ this(null, null, null, null, null);
+ }
+
+ public TimelineCollectorContext(String clusterId, String userId,
+ String flowId, String flowRunId, String appId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public void setClusterId(String clusterId) {
+ this.clusterId = clusterId;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public void setUserId(String userId) {
+ this.userId = userId;
+ }
+
+ public String getFlowId() {
+ return flowId;
+ }
+
+ public void setFlowId(String flowId) {
+ this.flowId = flowId;
+ }
+
+ public String getFlowRunId() {
+ return flowRunId;
+ }
+
+ public void setFlowRunId(String flowRunId) {
+ this.flowRunId = flowRunId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public void setAppId(String appId) {
+ this.appId = appId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8b5ab64/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 3a4515e..909027e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -43,6 +43,8 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
@@ -102,6 +104,7 @@ public class TimelineCollectorManager extends CompositeService {
@Override
protected void serviceStart() throws Exception {
+ nmCollectorService = getNMCollectorService();
startWebApp();
super.serviceStart();
}
@@ -151,11 +154,11 @@ public class TimelineCollectorManager extends CompositeService {
// Report to NM if a new collector is added.
if (collectorIsNew) {
try {
+ updateTimelineCollectorContext(appId, collector);
reportNewCollectorToNM(appId);
} catch (Exception e) {
- // throw exception here as it cannot be used if failed report to NM
- LOG.error("Failed to report a new collector for application: " + appId +
- " to the NM Collector Service.");
+ // throw exception here as it cannot be used if failed communicate with NM
+ LOG.error("Failed to communicate with NM Collector Service for " + appId);
throw new YarnRuntimeException(e);
}
}
@@ -250,7 +253,6 @@ public class TimelineCollectorManager extends CompositeService {
private void reportNewCollectorToNM(ApplicationId appId)
throws YarnException, IOException {
- this.nmCollectorService = getNMCollectorService();
ReportNewCollectorInfoRequest request =
ReportNewCollectorInfoRequest.newInstance(appId,
this.timelineRestServerBindAddress);
@@ -259,6 +261,28 @@ public class TimelineCollectorManager extends CompositeService {
nmCollectorService.reportNewCollectorInfo(request);
}
+ private void updateTimelineCollectorContext(
+ ApplicationId appId, TimelineCollector collector)
+ throws YarnException, IOException {
+ GetTimelineCollectorContextRequest request =
+ GetTimelineCollectorContextRequest.newInstance(appId);
+ LOG.info("Get timeline collector context for " + appId);
+ GetTimelineCollectorContextResponse response =
+ nmCollectorService.getTimelineCollectorContext(request);
+ String userId = response.getUserId();
+ if (userId != null && !userId.isEmpty()) {
+ collector.getTimelineEntityContext().setUserId(userId);
+ }
+ String flowId = response.getFlowId();
+ if (flowId != null && !flowId.isEmpty()) {
+ collector.getTimelineEntityContext().setFlowId(flowId);
+ }
+ String flowRunId = response.getFlowRunId();
+ if (flowRunId != null && !flowRunId.isEmpty()) {
+ collector.getTimelineEntityContext().setFlowRunId(flowRunId);
+ }
+ }
+
@VisibleForTesting
protected CollectorNodemanagerProtocol getNMCollectorService() {
Configuration conf = getConfig();
[22/50] [abbrv] hadoop git commit: YARN-3726. Fix
TestHBaseTimelineWriterImpl unit test failure by fixing its test data
(Vrushali C via sjlee)
Posted by sj...@apache.org.
YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its test data (Vrushali C via sjlee)
(cherry picked from commit 4c0b6d73914f2e249795deb292f508177ea54884)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0d1cc01
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0d1cc01
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0d1cc01
Branch: refs/heads/YARN-2928
Commit: d0d1cc014707ee99f53d902248eb77a5ec6f3bcc
Parents: b3edd63
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed May 27 20:28:04 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:12 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 ++++
.../storage/TestHBaseTimelineWriterImpl.java | 15 ++++++++-------
2 files changed, 12 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0d1cc01/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3df01e9..bc3880c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -75,6 +75,10 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3411. [Storage implementation] explore the native HBase write schema
for storage (Vrushali C via sjlee)
+
+ YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
+ test data (Vrushali C via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0d1cc01/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 48bacd6..f999b4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -121,12 +121,13 @@ public class TestHBaseTimelineWriterImpl {
TimelineMetric m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
Map<Long, Number> metricValues = new HashMap<Long, Number>();
- metricValues.put(1429741609000L, 100000000);
- metricValues.put(1429742609000L, 200000000);
- metricValues.put(1429743609000L, 300000000);
- metricValues.put(1429744609000L, 400000000);
- metricValues.put(1429745609000L, 50000000000L);
- metricValues.put(1429746609000L, 60000000000L);
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
@@ -216,7 +217,7 @@ public class TestHBaseTimelineWriterImpl {
private void checkMetricsTimeseries(List<Cell> metricCells,
TimelineMetric m1) throws IOException {
Map<Long, Number> timeseries = m1.getValues();
- assertEquals(metricCells.size(), timeseries.size());
+ assertEquals(timeseries.size(), metricCells.size());
for (Cell c1 : metricCells) {
assertTrue(timeseries.containsKey(c1.getTimestamp()));
assertEquals(GenericObjectMapper.read(CellUtil.cloneValue(c1)),
[25/50] [abbrv] hadoop git commit: YARN-3044. Made RM write app,
attempt and optional container lifecycle events to timeline service
v2. Contributed by Naganarasimha G R.
Posted by sj...@apache.org.
YARN-3044. Made RM write app, attempt and optional container lifecycle events to timeline service v2. Contributed by Naganarasimha G R.
(cherry picked from commit 17842a3f61ed33ec831a889c11a74d4814be73ec)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a22d26c6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a22d26c6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a22d26c6
Branch: refs/heads/YARN-2928
Commit: a22d26c6d39f8967fc1b761ad47d21c76019c501
Parents: 6dd2397
Author: Zhijie Shen <zj...@apache.org>
Authored: Sat Jun 13 11:32:41 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:12 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../distributedshell/TestDistributedShell.java | 118 +++--
.../metrics/ContainerMetricsConstants.java | 10 +
.../server/resourcemanager/ResourceManager.java | 2 +-
.../AbstractTimelineServicePublisher.java | 179 ++++++++
.../metrics/ApplicationFinishedEvent.java | 13 +-
.../metrics/SystemMetricsPublisher.java | 431 ++++---------------
.../metrics/TimelineServiceV1Publisher.java | 270 ++++++++++++
.../metrics/TimelineServiceV2Publisher.java | 301 +++++++++++++
.../server/resourcemanager/rmapp/RMAppImpl.java | 2 -
.../TestRMAppLogAggregationStatus.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 8 +-
.../TestSystemMetricsPublisherForV2.java | 374 ++++++++++++++++
.../storage/FileSystemTimelineWriterImpl.java | 4 +-
16 files changed, 1331 insertions(+), 399 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b3e8738..a673fb2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -82,6 +82,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3721. build is broken on YARN-2928 branch due to possible dependency
cycle (Li Lu via sjlee)
+ YARN-3044. Made RM write app, attempt and optional container lifecycle
+ events to timeline service v2. (Naganarasimha G R via zjshen)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index defadec..a641f32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -470,4 +470,7 @@ public class TimelineEntity {
return real == null ? this : real;
}
+ public String toString() {
+ return identifier.toString();
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index e26b13f..8640864 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -388,6 +388,16 @@ public class YarnConfiguration extends Configuration {
+ "system-metrics-publisher.enabled";
public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
+ /**
+ * The setting that controls whether yarn container metrics is published to
+ * the timeline server or not by RM. This configuration setting is for ATS
+ * V2
+ */
+ public static final String RM_PUBLISH_CONTAINER_METRICS_ENABLED = YARN_PREFIX
+ + "rm.system-metrics-publisher.emit-container-events";
+ public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED =
+ false;
+
public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size";
public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index cc5f5e2..d31f2c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -52,13 +52,14 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -107,6 +108,7 @@ public class TestDistributedShell {
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
+ ".class", PerNodeTimelineCollectorsAuxService.class.getName());
}
+ conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
// Enable ContainersMonitorImpl
@@ -373,48 +375,96 @@ public class TestDistributedShell {
"/1/1/" : "/test_flow_name/test_flow_version/12345678/") +
appId.toString();
// for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
- String outputDirApp = basePath + "/DS_APP_ATTEMPT/";
-
- File entityFolder = new File(outputDirApp);
- Assert.assertTrue(entityFolder.isDirectory());
+ // Verify DS_APP_ATTEMPT entities posted by the client
// there will be at least one attempt, look for that file
- String appTimestampFileName = "appattempt_" + appId.getClusterTimestamp()
- + "_000" + appId.getId() + "_000001"
- + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
- String appAttemptFileName = outputDirApp + appTimestampFileName;
- File appAttemptFile = new File(appAttemptFileName);
- Assert.assertTrue(appAttemptFile.exists());
-
- String outputDirContainer = basePath + "/DS_CONTAINER/";
- File containerFolder = new File(outputDirContainer);
- Assert.assertTrue(containerFolder.isDirectory());
-
- String containerTimestampFileName = "container_"
- + appId.getClusterTimestamp() + "_000" + appId.getId()
- + "_01_000002.thist";
- String containerFileName = outputDirContainer + containerTimestampFileName;
- File containerFile = new File(containerFileName);
- Assert.assertTrue(containerFile.exists());
+ String appTimestampFileName =
+ "appattempt_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ verifyEntityTypeFileExists(basePath, "DS_APP_ATTEMPT",
+ appTimestampFileName);
+
+ // Verify DS_CONTAINER entities posted by the client
+ String containerTimestampFileName =
+ "container_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_01_000002.thist";
+ verifyEntityTypeFileExists(basePath, "DS_CONTAINER",
+ containerTimestampFileName);
// Verify NM posting container metrics info.
- String outputDirContainerMetrics = basePath + "/" +
- TimelineEntityType.YARN_CONTAINER + "/";
- File containerMetricsFolder = new File(outputDirContainerMetrics);
- Assert.assertTrue(containerMetricsFolder.isDirectory());
+ String containerMetricsTimestampFileName =
+ "container_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_01_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ verifyEntityTypeFileExists(basePath,
+ TimelineEntityType.YARN_CONTAINER.toString(),
+ containerMetricsTimestampFileName);
+
+ // Verify RM posting Application life cycle Events are getting published
+ String appMetricsTimestampFileName =
+ "application_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appEntityFile =
+ verifyEntityTypeFileExists(basePath,
+ TimelineEntityType.YARN_APPLICATION.toString(),
+ appMetricsTimestampFileName);
+ verifyStringExistsSpecifiedTimes(appEntityFile,
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE, 1,
+ "Application created event should be published atleast once");
+ verifyStringExistsSpecifiedTimes(appEntityFile,
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE, 1,
+ "Application finished event should be published atleast once");
+
+ // Verify RM posting AppAttempt life cycle Events are getting published
+ String appAttemptMetricsTimestampFileName =
+ "appattempt_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_000001"
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appAttemptEntityFile =
+ verifyEntityTypeFileExists(basePath,
+ TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
+ appAttemptMetricsTimestampFileName);
+ verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
+ AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE, 1,
+ "AppAttempt register event should be published atleast once");
+ verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
+ AppAttemptMetricsConstants.FINISHED_EVENT_TYPE, 1,
+ "AppAttempt finished event should be published atleast once");
+ } finally {
+ FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
+ }
+ }
- String containerMetricsTimestampFileName = "container_"
- + appId.getClusterTimestamp() + "_000" + appId.getId()
- + "_01_000001.thist";
- String containerMetricsFileName = outputDirContainerMetrics +
- containerMetricsTimestampFileName;
+ private File verifyEntityTypeFileExists(String basePath, String entityType,
+ String entityfileName) {
+ String outputDirPathForEntity = basePath + "/" + entityType + "/";
+ File outputDirForEntity = new File(outputDirPathForEntity);
+ Assert.assertTrue(outputDirForEntity.isDirectory());
- File containerMetricsFile = new File(containerMetricsFileName);
- Assert.assertTrue(containerMetricsFile.exists());
+ String entityFilePath = outputDirPathForEntity + entityfileName;
+ File entityFile = new File(entityFilePath);
+ Assert.assertTrue(entityFile.exists());
+ return entityFile;
+ }
+
+ private void verifyStringExistsSpecifiedTimes(File entityFile,
+ String searchString, long expectedNumOfTimes, String errorMsg)
+ throws IOException {
+ BufferedReader reader = null;
+ String strLine;
+ long actualCount = 0;
+ try {
+ reader = new BufferedReader(new FileReader(entityFile));
+ while ((strLine = reader.readLine()) != null) {
+ if (strLine.trim().contains(searchString))
+ actualCount++;
+ }
} finally {
- FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
+ reader.close();
}
+ Assert.assertEquals(errorMsg, expectedNumOfTimes, actualCount);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
index 0d5540d..7b42994 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
@@ -27,10 +27,20 @@ public class ContainerMetricsConstants {
public static final String ENTITY_TYPE = "YARN_CONTAINER";
+ // Event of this type will be emitted by NM.
public static final String CREATED_EVENT_TYPE = "YARN_CONTAINER_CREATED";
+ // Event of this type will be emitted by RM.
+ public static final String CREATED_IN_RM_EVENT_TYPE =
+ "YARN_RM_CONTAINER_CREATED";
+
+ // Event of this type will be emitted by NM.
public static final String FINISHED_EVENT_TYPE = "YARN_CONTAINER_FINISHED";
+ // Event of this type will be emitted by RM.
+ public static final String FINISHED_IN_RM_EVENT_TYPE =
+ "YARN_RM_CONTAINER_FINISHED";
+
public static final String PARENT_PRIMARIY_FILTER = "YARN_CONTAINER_PARENT";
public static final String ALLOCATED_MEMORY_ENTITY_INFO =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 525c88b..70c0014 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -373,7 +373,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected SystemMetricsPublisher createSystemMetricsPublisher() {
- return new SystemMetricsPublisher();
+ return new SystemMetricsPublisher(rmContext);
}
// sanity check for configurations
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java
new file mode 100644
index 0000000..0d66327
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractTimelineServicePublisher.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.Event;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher.TimelineServicePublisher;
+
+public abstract class AbstractTimelineServicePublisher extends CompositeService
+ implements TimelineServicePublisher, EventHandler<SystemMetricsEvent> {
+
+ private static final Log LOG = LogFactory
+ .getLog(TimelineServiceV2Publisher.class);
+
+ private Configuration conf;
+
+ public AbstractTimelineServicePublisher(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ this.conf = conf;
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ @Override
+ public void handle(SystemMetricsEvent event) {
+ switch (event.getType()) {
+ case APP_CREATED:
+ publishApplicationCreatedEvent((ApplicationCreatedEvent) event);
+ break;
+ case APP_FINISHED:
+ publishApplicationFinishedEvent((ApplicationFinishedEvent) event);
+ break;
+ case APP_ACLS_UPDATED:
+ publishApplicationACLsUpdatedEvent((ApplicationACLsUpdatedEvent) event);
+ break;
+ case APP_ATTEMPT_REGISTERED:
+ publishAppAttemptRegisteredEvent((AppAttemptRegisteredEvent) event);
+ break;
+ case APP_ATTEMPT_FINISHED:
+ publishAppAttemptFinishedEvent((AppAttemptFinishedEvent) event);
+ break;
+ case CONTAINER_CREATED:
+ publishContainerCreatedEvent((ContainerCreatedEvent) event);
+ break;
+ case CONTAINER_FINISHED:
+ publishContainerFinishedEvent((ContainerFinishedEvent) event);
+ break;
+ default:
+ LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
+ }
+ }
+
+ abstract void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event);
+
+ abstract void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event);
+
+ abstract void publishApplicationACLsUpdatedEvent(
+ ApplicationACLsUpdatedEvent event);
+
+ abstract void publishApplicationFinishedEvent(ApplicationFinishedEvent event);
+
+ abstract void publishApplicationCreatedEvent(ApplicationCreatedEvent event);
+
+ abstract void publishContainerCreatedEvent(ContainerCreatedEvent event);
+
+ abstract void publishContainerFinishedEvent(ContainerFinishedEvent event);
+
+ @Override
+ public Dispatcher getDispatcher() {
+ MultiThreadedDispatcher dispatcher =
+ new MultiThreadedDispatcher(
+ conf.getInt(
+ YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
+ YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE));
+ dispatcher.setDrainEventsOnStop();
+ return dispatcher;
+ }
+
+ @Override
+ public boolean publishRMContainerMetrics() {
+ return true;
+ }
+
+ @Override
+ public EventHandler<SystemMetricsEvent> getEventHandler() {
+ return this;
+ }
+
+ @SuppressWarnings({ "rawtypes", "unchecked" })
+ public static class MultiThreadedDispatcher extends CompositeService
+ implements Dispatcher {
+
+ private List<AsyncDispatcher> dispatchers =
+ new ArrayList<AsyncDispatcher>();
+
+ public MultiThreadedDispatcher(int num) {
+ super(MultiThreadedDispatcher.class.getName());
+ for (int i = 0; i < num; ++i) {
+ AsyncDispatcher dispatcher = createDispatcher();
+ dispatchers.add(dispatcher);
+ addIfService(dispatcher);
+ }
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ return new CompositEventHandler();
+ }
+
+ @Override
+ public void register(Class<? extends Enum> eventType, EventHandler handler) {
+ for (AsyncDispatcher dispatcher : dispatchers) {
+ dispatcher.register(eventType, handler);
+ }
+ }
+
+ public void setDrainEventsOnStop() {
+ for (AsyncDispatcher dispatcher : dispatchers) {
+ dispatcher.setDrainEventsOnStop();
+ }
+ }
+
+ private class CompositEventHandler implements EventHandler<Event> {
+
+ @Override
+ public void handle(Event event) {
+ // Use hashCode (of ApplicationId) to dispatch the event to the child
+ // dispatcher, such that all the writing events of one application will
+ // be handled by one thread, the scheduled order of the these events
+ // will be preserved
+ int index = (event.hashCode() & Integer.MAX_VALUE) % dispatchers.size();
+ dispatchers.get(index).getEventHandler().handle(event);
+ }
+ }
+
+ protected AsyncDispatcher createDispatcher() {
+ return new AsyncDispatcher();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
index 8d75f92..d9241b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ApplicationFinishedEvent.java
@@ -22,6 +22,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
public class ApplicationFinishedEvent extends
@@ -33,6 +35,7 @@ public class ApplicationFinishedEvent extends
private YarnApplicationState state;
private ApplicationAttemptId latestAppAttemptId;
private RMAppMetrics appMetrics;
+ private RMAppImpl app;
public ApplicationFinishedEvent(
ApplicationId appId,
@@ -41,14 +44,16 @@ public class ApplicationFinishedEvent extends
YarnApplicationState state,
ApplicationAttemptId latestAppAttemptId,
long finishedTime,
- RMAppMetrics appMetrics) {
+ RMAppMetrics appMetrics,
+ RMAppImpl app) {
super(SystemMetricsEventType.APP_FINISHED, finishedTime);
this.appId = appId;
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.latestAppAttemptId = latestAppAttemptId;
this.state = state;
- this.appMetrics=appMetrics;
+ this.appMetrics = appMetrics;
+ this.app = app;
}
@Override
@@ -56,6 +61,10 @@ public class ApplicationFinishedEvent extends
return appId.hashCode();
}
+ public RMAppImpl getApp() {
+ return app;
+ }
+
public ApplicationId getApplicationId() {
return appId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index b4ce4f9..1b8cce3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -29,31 +25,23 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+import com.google.common.annotations.VisibleForTesting;
/**
- * The class that helps RM publish metrics to the timeline server V1. RM will
+ * The class that helps RM publish metrics to the timeline server. RM will
* always invoke the methods of this class regardless the service is enabled or
* not. If it is disabled, publishing requests will be ignored silently.
*/
@@ -65,30 +53,38 @@ public class SystemMetricsPublisher extends CompositeService {
.getLog(SystemMetricsPublisher.class);
private Dispatcher dispatcher;
- private TimelineClient client;
- private boolean publishSystemMetricsToATSv1;
+ private boolean publishSystemMetrics;
+ private boolean publishContainerMetrics;
+ protected RMContext rmContext;
- public SystemMetricsPublisher() {
+ public SystemMetricsPublisher(RMContext rmContext) {
super(SystemMetricsPublisher.class.getName());
+ this.rmContext = rmContext;
}
@Override
protected void serviceInit(Configuration conf) throws Exception {
- publishSystemMetricsToATSv1 =
+ publishSystemMetrics =
conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
- && conf.getBoolean(
- YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
- YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
-
- if (publishSystemMetricsToATSv1) {
- client = TimelineClient.createTimelineClient();
- addIfService(client);
-
- dispatcher = createDispatcher(conf);
- dispatcher.register(SystemMetricsEventType.class,
- new ForwardingEventHandler());
- addIfService(dispatcher);
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
+ if (publishSystemMetrics) {
+ TimelineServicePublisher timelineServicePublisher =
+ getTimelineServicePublisher(conf);
+ if (timelineServicePublisher != null) {
+ addService(timelineServicePublisher);
+ // init required to be called so that other methods of
+ // TimelineServicePublisher can be utilized
+ timelineServicePublisher.init(conf);
+ dispatcher = createDispatcher(timelineServicePublisher);
+ publishContainerMetrics =
+ timelineServicePublisher.publishRMContainerMetrics();
+ dispatcher.register(SystemMetricsEventType.class,
+ timelineServicePublisher.getEventHandler());
+ addIfService(dispatcher);
+ } else {
+ LOG.info("TimelineServicePublisher is not configured");
+ publishSystemMetrics = false;
+ }
LOG.info("YARN system metrics publishing service is enabled");
} else {
LOG.info("YARN system metrics publishing service is not enabled");
@@ -96,9 +92,26 @@ public class SystemMetricsPublisher extends CompositeService {
super.serviceInit(conf);
}
+ @VisibleForTesting
+ Dispatcher createDispatcher(TimelineServicePublisher timelineServicePublisher) {
+ return timelineServicePublisher.getDispatcher();
+ }
+
+ TimelineServicePublisher getTimelineServicePublisher(Configuration conf) {
+ if (conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED)) {
+ return new TimelineServiceV1Publisher();
+ } else if (conf.getBoolean(
+ YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED)) {
+ return new TimelineServiceV2Publisher(rmContext);
+ }
+ return null;
+ }
+
@SuppressWarnings("unchecked")
public void appCreated(RMApp app, long createdTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new ApplicationCreatedEvent(
app.getApplicationId(),
@@ -115,7 +128,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appFinished(RMApp app, RMAppState state, long finishedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new ApplicationFinishedEvent(
app.getApplicationId(),
@@ -125,14 +138,15 @@ public class SystemMetricsPublisher extends CompositeService {
app.getCurrentAppAttempt() == null ?
null : app.getCurrentAppAttempt().getAppAttemptId(),
finishedTime,
- app.getRMAppMetrics()));
+ app.getRMAppMetrics(),
+ (RMAppImpl)app));
}
}
@SuppressWarnings("unchecked")
public void appACLsUpdated(RMApp app, String appViewACLs,
long updatedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new ApplicationACLsUpdatedEvent(
app.getApplicationId(),
@@ -144,7 +158,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appAttemptRegistered(RMAppAttempt appAttempt,
long registeredTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new AppAttemptRegisteredEvent(
appAttempt.getAppAttemptId(),
@@ -160,7 +174,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishSystemMetrics) {
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
appAttempt.getAppAttemptId(),
@@ -177,7 +191,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void containerCreated(RMContainer container, long createdTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishContainerMetrics) {
dispatcher.getEventHandler().handle(
new ContainerCreatedEvent(
container.getContainerId(),
@@ -190,7 +204,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void containerFinished(RMContainer container, long finishedTime) {
- if (publishSystemMetricsToATSv1) {
+ if (publishContainerMetrics) {
dispatcher.getEventHandler().handle(
new ContainerFinishedEvent(
container.getContainerId(),
@@ -201,322 +215,31 @@ public class SystemMetricsPublisher extends CompositeService {
}
}
- protected Dispatcher createDispatcher(Configuration conf) {
- MultiThreadedDispatcher dispatcher =
- new MultiThreadedDispatcher(
- conf.getInt(
- YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
- YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE));
- dispatcher.setDrainEventsOnStop();
- return dispatcher;
- }
-
- protected void handleSystemMetricsEvent(
- SystemMetricsEvent event) {
- switch (event.getType()) {
- case APP_CREATED:
- publishApplicationCreatedEvent((ApplicationCreatedEvent) event);
- break;
- case APP_FINISHED:
- publishApplicationFinishedEvent((ApplicationFinishedEvent) event);
- break;
- case APP_ACLS_UPDATED:
- publishApplicationACLsUpdatedEvent((ApplicationACLsUpdatedEvent) event);
- break;
- case APP_ATTEMPT_REGISTERED:
- publishAppAttemptRegisteredEvent((AppAttemptRegisteredEvent) event);
- break;
- case APP_ATTEMPT_FINISHED:
- publishAppAttemptFinishedEvent((AppAttemptFinishedEvent) event);
- break;
- case CONTAINER_CREATED:
- publishContainerCreatedEvent((ContainerCreatedEvent) event);
- break;
- case CONTAINER_FINISHED:
- publishContainerFinishedEvent((ContainerFinishedEvent) event);
- break;
- default:
- LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
- }
- }
-
- private void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
- TimelineEntity entity =
- createApplicationEntity(event.getApplicationId());
- Map<String, Object> entityInfo = new HashMap<String, Object>();
- entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
- event.getApplicationName());
- entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
- event.getApplicationType());
- entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
- event.getUser());
- entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
- event.getQueue());
- entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
- event.getSubmittedTime());
- entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
- event.getAppTags());
- entityInfo.put(
- ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
- event.isUnmanagedApp());
- entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
- event.getApplicationPriority().getPriority());
- entity.setOtherInfo(entityInfo);
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(
- ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- entity.addEvent(tEvent);
- putEntity(entity);
+ @VisibleForTesting
+ boolean isPublishContainerMetrics() {
+ return publishContainerMetrics;
}
- private void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
- TimelineEntity entity =
- createApplicationEntity(event.getApplicationId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
- event.getDiagnosticsInfo());
- eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO,
- event.getFinalApplicationStatus().toString());
- eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO,
- event.getYarnApplicationState().toString());
- if (event.getLatestApplicationAttemptId() != null) {
- eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
- event.getLatestApplicationAttemptId().toString());
- }
- RMAppMetrics appMetrics = event.getAppMetrics();
- entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
- appMetrics.getVcoreSeconds());
- entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
- appMetrics.getMemorySeconds());
-
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishApplicationACLsUpdatedEvent(
- ApplicationACLsUpdatedEvent event) {
- TimelineEntity entity =
- createApplicationEntity(event.getApplicationId());
- TimelineEvent tEvent = new TimelineEvent();
- Map<String, Object> entityInfo = new HashMap<String, Object>();
- entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
- event.getViewAppACLs());
- entity.setOtherInfo(entityInfo);
- tEvent.setEventType(
- ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private static TimelineEntity createApplicationEntity(
- ApplicationId applicationId) {
- TimelineEntity entity = new TimelineEntity();
- entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(applicationId.toString());
- return entity;
- }
-
- private void
- publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
- TimelineEntity entity =
- createAppAttemptEntity(event.getApplicationAttemptId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(
- AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(
- AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
- event.getTrackingUrl());
- eventInfo.put(
- AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
- event.getOriginalTrackingURL());
- eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO,
- event.getHost());
- eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
- event.getRpcPort());
- eventInfo.put(
- AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
- event.getMasterContainerId().toString());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
- TimelineEntity entity =
- createAppAttemptEntity(event.getApplicationAttemptId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(
- AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
- event.getTrackingUrl());
- eventInfo.put(
- AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
- event.getOriginalTrackingURL());
- eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
- event.getDiagnosticsInfo());
- eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO,
- event.getFinalApplicationStatus().toString());
- eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
- event.getYarnApplicationAttemptState().toString());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private static TimelineEntity createAppAttemptEntity(
- ApplicationAttemptId appAttemptId) {
- TimelineEntity entity = new TimelineEntity();
- entity.setEntityType(
- AppAttemptMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(appAttemptId.toString());
- entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
- appAttemptId.getApplicationId().toString());
- return entity;
- }
-
- private void publishContainerCreatedEvent(ContainerCreatedEvent event) {
- TimelineEntity entity = createContainerEntity(event.getContainerId());
- Map<String, Object> entityInfo = new HashMap<String, Object>();
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
- event.getAllocatedResource().getMemory());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO,
- event.getAllocatedResource().getVirtualCores());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
- event.getAllocatedNode().getHost());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
- event.getAllocatedNode().getPort());
- entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
- event.getAllocatedPriority().getPriority());
- entityInfo.put(
- ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
- event.getNodeHttpAddress());
- entity.setOtherInfo(entityInfo);
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private void publishContainerFinishedEvent(ContainerFinishedEvent event) {
- TimelineEntity entity = createContainerEntity(event.getContainerId());
- TimelineEvent tEvent = new TimelineEvent();
- tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
- tEvent.setTimestamp(event.getTimestamp());
- Map<String, Object> eventInfo = new HashMap<String, Object>();
- eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
- event.getDiagnosticsInfo());
- eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
- event.getContainerExitStatus());
- eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO,
- event.getContainerState().toString());
- tEvent.setEventInfo(eventInfo);
- entity.addEvent(tEvent);
- putEntity(entity);
- }
-
- private static TimelineEntity createContainerEntity(
- ContainerId containerId) {
- TimelineEntity entity = new TimelineEntity();
- entity.setEntityType(
- ContainerMetricsConstants.ENTITY_TYPE);
- entity.setEntityId(containerId.toString());
- entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
- containerId.getApplicationAttemptId().toString());
- return entity;
- }
-
- private void putEntity(TimelineEntity entity) {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Publishing the entity " + entity.getEntityId() +
- ", JSON-style content: " + TimelineUtils.dumpTimelineRecordtoJSON(entity));
- }
- client.putEntities(entity);
- } catch (Exception e) {
- LOG.error("Error when publishing entity [" + entity.getEntityType() + ","
- + entity.getEntityId() + "]", e);
- }
+ @VisibleForTesting
+ Dispatcher getDispatcher() {
+ return dispatcher;
}
- /**
- * EventHandler implementation which forward events to SystemMetricsPublisher.
- * Making use of it, SystemMetricsPublisher can avoid to have a public handle
- * method.
- */
- private final class ForwardingEventHandler implements
- EventHandler<SystemMetricsEvent> {
+ interface TimelineServicePublisher extends Service {
+ /**
+ * @return the Dispatcher which needs to be used to dispatch events
+ */
+ Dispatcher getDispatcher();
- @Override
- public void handle(SystemMetricsEvent event) {
- handleSystemMetricsEvent(event);
- }
+ /**
+ * @return true if RMContainerMetricsNeeds to be sent
+ */
+ boolean publishRMContainerMetrics();
+ /**
+ * @return EventHandler which needs to be registered to the dispatcher to
+ * handle the SystemMetricsEvent
+ */
+ EventHandler<SystemMetricsEvent> getEventHandler();
}
-
- @SuppressWarnings({ "rawtypes", "unchecked" })
- protected static class MultiThreadedDispatcher extends CompositeService
- implements Dispatcher {
-
- private List<AsyncDispatcher> dispatchers =
- new ArrayList<AsyncDispatcher>();
-
- public MultiThreadedDispatcher(int num) {
- super(MultiThreadedDispatcher.class.getName());
- for (int i = 0; i < num; ++i) {
- AsyncDispatcher dispatcher = createDispatcher();
- dispatchers.add(dispatcher);
- addIfService(dispatcher);
- }
- }
-
- @Override
- public EventHandler getEventHandler() {
- return new CompositEventHandler();
- }
-
- @Override
- public void register(Class<? extends Enum> eventType, EventHandler handler) {
- for (AsyncDispatcher dispatcher : dispatchers) {
- dispatcher.register(eventType, handler);
- }
- }
-
- public void setDrainEventsOnStop() {
- for (AsyncDispatcher dispatcher : dispatchers) {
- dispatcher.setDrainEventsOnStop();
- }
- }
-
- private class CompositEventHandler implements EventHandler<Event> {
-
- @Override
- public void handle(Event event) {
- // Use hashCode (of ApplicationId) to dispatch the event to the child
- // dispatcher, such that all the writing events of one application will
- // be handled by one thread, the scheduled order of the these events
- // will be preserved
- int index = (event.hashCode() & Integer.MAX_VALUE) % dispatchers.size();
- dispatchers.get(index).getEventHandler().handle(event);
- }
-
- }
-
- protected AsyncDispatcher createDispatcher() {
- return new AsyncDispatcher();
- }
-
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
new file mode 100644
index 0000000..c8db8ff
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -0,0 +1,270 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+public class TimelineServiceV1Publisher extends
+ AbstractTimelineServicePublisher {
+
+ private static final Log LOG = LogFactory
+ .getLog(TimelineServiceV1Publisher.class);
+
+ public TimelineServiceV1Publisher() {
+ super("TimelineserviceV1Publisher");
+ }
+
+ private TimelineClient client;
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ client = TimelineClient.createTimelineClient();
+ addIfService(client);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
+ event.getApplicationName());
+ entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
+ event.getApplicationType());
+ entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
+ event.getUser());
+ entityInfo.put(ApplicationMetricsConstants.QUEUE_ENTITY_INFO,
+ event.getQueue());
+ entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
+ event.getSubmittedTime());
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
+ event.getAppTags());
+ entityInfo.put(
+ ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
+ event.isUnmanagedApp());
+ entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+ event.getApplicationPriority().getPriority());
+ entity.setOtherInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationState().toString());
+ if (event.getLatestApplicationAttemptId() != null) {
+ eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
+ event.getLatestApplicationAttemptId().toString());
+ }
+ RMAppMetrics appMetrics = event.getAppMetrics();
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
+ appMetrics.getVcoreSeconds());
+ entity.addOtherInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
+ appMetrics.getMemorySeconds());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishApplicationACLsUpdatedEvent(ApplicationACLsUpdatedEvent event) {
+ TimelineEntity entity = createApplicationEntity(event.getApplicationId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
+ event.getViewAppACLs());
+ entity.setOtherInfo(entityInfo);
+ tEvent.setEventType(ApplicationMetricsConstants.ACLS_UPDATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ private static TimelineEntity createApplicationEntity(
+ ApplicationId applicationId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(ApplicationMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(applicationId.toString());
+ return entity;
+ }
+
+ @Override
+ void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
+ TimelineEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, event.getHost());
+ eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
+ event.getRpcPort());
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event
+ .getMasterContainerId().toString());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
+ TimelineEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationAttemptState().toString());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ private static TimelineEntity createAppAttemptEntity(
+ ApplicationAttemptId appAttemptId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(AppAttemptMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(appAttemptId.toString());
+ entity.addPrimaryFilter(AppAttemptMetricsConstants.PARENT_PRIMARY_FILTER,
+ appAttemptId.getApplicationId().toString());
+ return entity;
+ }
+
+ @Override
+ void publishContainerCreatedEvent(ContainerCreatedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
+ event.getAllocatedResource().getMemory());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, event
+ .getAllocatedResource().getVirtualCores());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, event
+ .getAllocatedNode().getHost());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, event
+ .getAllocatedNode().getPort());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
+ event.getAllocatedPriority().getPriority());
+ entityInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ event.getNodeHttpAddress());
+ entity.setOtherInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ @Override
+ void publishContainerFinishedEvent(ContainerFinishedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setEventType(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
+ event.getContainerExitStatus());
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, event
+ .getContainerState().toString());
+ tEvent.setEventInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity);
+ }
+
+ private static TimelineEntity createContainerEntity(ContainerId containerId) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setEntityType(ContainerMetricsConstants.ENTITY_TYPE);
+ entity.setEntityId(containerId.toString());
+ entity.addPrimaryFilter(ContainerMetricsConstants.PARENT_PRIMARIY_FILTER,
+ containerId.getApplicationAttemptId().toString());
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Publishing the entity " + entity.getEntityId()
+ + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ client.putEntities(entity);
+ } catch (Exception e) {
+ LOG.error("Error when publishing entity [" + entity.getEntityType() + ","
+ + entity.getEntityId() + "]", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
new file mode 100644
index 0000000..700708a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+/**
+ * This class is responsible for posting application, appattempt & Container
+ * lifecycle related events to timeline service V2
+ */
+@Private
+@Unstable
+public class TimelineServiceV2Publisher extends
+ AbstractTimelineServicePublisher {
+ private static final Log LOG = LogFactory
+ .getLog(TimelineServiceV2Publisher.class);
+ protected RMTimelineCollectorManager rmTimelineCollectorManager;
+
+ public TimelineServiceV2Publisher(RMContext rmContext) {
+ super("TimelineserviceV2Publisher");
+ rmTimelineCollectorManager = rmContext.getRMTimelineCollectorManager();
+ }
+
+ private boolean publishContainerMetrics;
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ publishContainerMetrics =
+ conf.getBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ void publishApplicationCreatedEvent(ApplicationCreatedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ entity.setQueue(event.getQueue());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.NAME_ENTITY_INFO,
+ event.getApplicationName());
+ entityInfo.put(ApplicationMetricsConstants.TYPE_ENTITY_INFO,
+ event.getApplicationType());
+ entityInfo.put(ApplicationMetricsConstants.USER_ENTITY_INFO,
+ event.getUser());
+ entityInfo.put(ApplicationMetricsConstants.SUBMITTED_TIME_ENTITY_INFO,
+ event.getSubmittedTime());
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO,
+ event.getAppTags());
+ entityInfo.put(
+ ApplicationMetricsConstants.UNMANAGED_APPLICATION_ENTITY_INFO,
+ event.isUnmanagedApp());
+ entityInfo.put(ApplicationMetricsConstants.APPLICATION_PRIORITY_INFO,
+ event.getApplicationPriority().getPriority());
+ entity.setInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ entity.addEvent(tEvent);
+
+ putEntity(entity, event.getApplicationId());
+ }
+
+ @Override
+ void publishApplicationFinishedEvent(ApplicationFinishedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ RMAppMetrics appMetrics = event.getAppMetrics();
+ entity.addInfo(ApplicationMetricsConstants.APP_CPU_METRICS,
+ appMetrics.getVcoreSeconds());
+ entity.addInfo(ApplicationMetricsConstants.APP_MEM_METRICS,
+ appMetrics.getMemorySeconds());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ApplicationMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ApplicationMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(ApplicationMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationState().toString());
+ if (event.getLatestApplicationAttemptId() != null) {
+ eventInfo.put(ApplicationMetricsConstants.LATEST_APP_ATTEMPT_EVENT_INFO,
+ event.getLatestApplicationAttemptId().toString());
+ }
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationId());
+
+ //cleaning up the collector cached
+ event.getApp().stopTimelineCollector();
+ }
+
+ @Override
+ void publishApplicationACLsUpdatedEvent(ApplicationACLsUpdatedEvent event) {
+ ApplicationEntity entity =
+ createApplicationEntity(event.getApplicationId());
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
+ event.getViewAppACLs());
+ entity.setInfo(entityInfo);
+
+ putEntity(entity, event.getApplicationId());
+ }
+
+ private static ApplicationEntity createApplicationEntity(
+ ApplicationId applicationId) {
+ ApplicationEntity entity = new ApplicationEntity();
+ entity.setId(applicationId.toString());
+ return entity;
+ }
+
+ @Override
+ void publishAppAttemptRegisteredEvent(AppAttemptRegisteredEvent event) {
+ TimelineEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.HOST_EVENT_INFO, event.getHost());
+ eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
+ event.getRpcPort());
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO, event
+ .getMasterContainerId().toString());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationAttemptId().getApplicationId());
+ }
+
+ @Override
+ void publishAppAttemptFinishedEvent(AppAttemptFinishedEvent event) {
+ ApplicationAttemptEntity entity =
+ createAppAttemptEntity(event.getApplicationAttemptId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(AppAttemptMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(AppAttemptMetricsConstants.TRACKING_URL_EVENT_INFO,
+ event.getTrackingUrl());
+ eventInfo.put(AppAttemptMetricsConstants.ORIGINAL_TRACKING_URL_EVENT_INFO,
+ event.getOriginalTrackingURL());
+ eventInfo.put(AppAttemptMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(AppAttemptMetricsConstants.FINAL_STATUS_EVENT_INFO, event
+ .getFinalApplicationStatus().toString());
+ eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO, event
+ .getYarnApplicationAttemptState().toString());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getApplicationAttemptId().getApplicationId());
+ }
+
+ @Override
+ void publishContainerCreatedEvent(ContainerCreatedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ // updated as event info instead of entity info, as entity info is updated
+ // by NM
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO, event
+ .getAllocatedResource().getMemory());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO, event
+ .getAllocatedResource().getVirtualCores());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO, event
+ .getAllocatedNode().getHost());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO, event
+ .getAllocatedNode().getPort());
+ eventInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
+ event.getAllocatedPriority().getPriority());
+ eventInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ event.getNodeHttpAddress());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getContainerId().getApplicationAttemptId()
+ .getApplicationId());
+ }
+
+ @Override
+ void publishContainerFinishedEvent(ContainerFinishedEvent event) {
+ TimelineEntity entity = createContainerEntity(event.getContainerId());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.FINISHED_IN_RM_EVENT_TYPE);
+ tEvent.setTimestamp(event.getTimestamp());
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ event.getDiagnosticsInfo());
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
+ event.getContainerExitStatus());
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, event
+ .getContainerState().toString());
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, event.getContainerId().getApplicationAttemptId()
+ .getApplicationId());
+ }
+
+ private static ContainerEntity createContainerEntity(ContainerId containerId) {
+ ContainerEntity entity = new ContainerEntity();
+ entity.setId(containerId.toString());
+ entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT
+ .name(), containerId.getApplicationAttemptId().toString()));
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity, ApplicationId appId) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ TimelineCollector timelineCollector =
+ rmTimelineCollectorManager.get(appId);
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ timelineCollector.putEntities(entities,
+ UserGroupInformation.getCurrentUser());
+ } catch (Exception e) {
+ LOG.error("Error when publishing entity " + entity, e);
+ }
+ }
+
+ private static ApplicationAttemptEntity createAppAttemptEntity(
+ ApplicationAttemptId appAttemptId) {
+ ApplicationAttemptEntity entity = new ApplicationAttemptEntity();
+ entity.setId(appAttemptId.toString());
+ entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION.name(),
+ appAttemptId.getApplicationId().toString()));
+ return entity;
+ }
+
+ @Override
+ public boolean publishRMContainerMetrics() {
+ return publishContainerMetrics;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index bbbbb93..261a520 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1301,8 +1301,6 @@ public class RMAppImpl implements RMApp, Recoverable {
.applicationFinished(app, finalState);
app.rmContext.getSystemMetricsPublisher()
.appFinished(app, finalState, app.finishTime);
-
- app.stopTimelineCollector();
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index ae14253..3af3350 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -89,7 +89,7 @@ public class TestRMAppLogAggregationStatus {
rmContext =
new RMContextImpl(rmDispatcher, null, null, null,
null, null, null, null, null);
- rmContext.setSystemMetricsPublisher(new SystemMetricsPublisher());
+ rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
rmContext
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index e0d1b6f..adb2259 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -46,7 +46,9 @@ import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistor
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -72,7 +74,7 @@ public class TestSystemMetricsPublisher {
public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
@@ -86,7 +88,7 @@ public class TestSystemMetricsPublisher {
timelineServer.start();
store = timelineServer.getTimelineStore();
- metricsPublisher = new SystemMetricsPublisher();
+ metricsPublisher = new SystemMetricsPublisher(mock(RMContext.class));
metricsPublisher.init(conf);
metricsPublisher.start();
}
@@ -351,7 +353,7 @@ public class TestSystemMetricsPublisher {
}
private static RMApp createRMApp(ApplicationId appId) {
- RMApp app = mock(RMApp.class);
+ RMApp app = mock(RMAppImpl.class);
when(app.getApplicationId()).thenReturn(appId);
when(app.getName()).thenReturn("test app");
when(app.getApplicationType()).thenReturn("test app type");
[09/50] [abbrv] hadoop git commit: YARN-3374. Collector's web server
should randomly bind an available port. Contributed by Zhijie Shen
Posted by sj...@apache.org.
YARN-3374. Collector's web server should randomly bind an available port. Contributed by Zhijie Shen
(cherry picked from commit 3aa898e734a1e4368ddf1d0bbd31f9b4de53ceba)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9c81ac5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9c81ac5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9c81ac5
Branch: refs/heads/YARN-2928
Commit: a9c81ac5729e3e2cb39210d4679a8da9ca5c4e4e
Parents: e8b5ab6
Author: Junping Du <ju...@apache.org>
Authored: Thu Apr 2 11:59:59 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:38:44 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/conf/YarnConfiguration.java | 1 +
.../collector/TimelineCollectorManager.java | 20 ++++++++++----------
.../collector/TestTimelineCollectorManager.java | 12 ++++++++++++
4 files changed, 26 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ed5dc88..76fa0a8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -44,6 +44,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3040. Make putEntities operation be aware of the app's context. (Zhijie Shen
via junping_du)
+ YARN-3374. Collector's web server should randomly bind an available port. (
+ Zhijie Shen via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 51f2b2d..1ba7f36 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1515,6 +1515,7 @@ public class YarnConfiguration extends Configuration {
/** The listening endpoint for the timeline service application.*/
public static final String TIMELINE_SERVICE_BIND_HOST =
TIMELINE_SERVICE_PREFIX + "bind-host";
+ public static final String DEFAULT_TIMELINE_SERVICE_BIND_HOST = "0.0.0.0";
/** The number of threads to handle client RPC API requests. */
public static final String TIMELINE_SERVICE_HANDLER_THREAD_COUNT =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 909027e..5f23c25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -210,22 +210,17 @@ public class TimelineCollectorManager extends CompositeService {
*/
private void startWebApp() {
Configuration conf = getConfig();
- // use the same ports as the old ATS for now; we could create new properties
- // for the new timeline service if needed
- String bindAddress = WebAppUtils.getWebAppBindURL(conf,
- YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
- WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
- this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
- NetUtils.createSocketAddr(bindAddress));
- LOG.info("Instantiating the per-node collector webapp at " +
- timelineRestServerBindAddress);
+ String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
try {
Configuration confForInfoServer = new Configuration(conf);
confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
HttpServer2.Builder builder = new HttpServer2.Builder()
.setName("timeline")
.setConf(conf)
- .addEndpoint(URI.create("http://" + bindAddress));
+ .addEndpoint(URI.create(
+ (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
+ bindAddress));
timelineRestServer = builder.build();
// TODO: replace this by an authentication filter in future.
HashMap<String, String> options = new HashMap<>();
@@ -249,6 +244,11 @@ public class TimelineCollectorManager extends CompositeService {
LOG.error(msg, e);
throw new YarnRuntimeException(msg, e);
}
+ //TODO: We need to think of the case of multiple interfaces
+ this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
+ timelineRestServer.getConnectorAddress(0));
+ LOG.info("Instantiated the per-node collector webapp at " +
+ timelineRestServerBindAddress);
}
private void reportNewCollectorToNM(ApplicationId appId)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9c81ac5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
index 38227ca..36bda85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.any;
@@ -63,6 +65,16 @@ public class TestTimelineCollectorManager {
}
}
+ @Test
+ public void testStartWebApp() throws Exception {
+ assertNotNull(collectorManager.getRestServerBindAddress());
+ String address = collectorManager.getRestServerBindAddress();
+ String[] parts = address.split(":");
+ assertEquals(2, parts.length);
+ assertNotNull(parts[0]);
+ assertTrue(Integer.valueOf(parts[1]) > 0);
+ }
+
@Test(timeout=60000)
public void testMultithreadedAdd() throws Exception {
final int NUM_APPS = 5;
[07/50] [abbrv] hadoop git commit: YARN-3333. Rename
TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java
new file mode 100644
index 0000000..3498de9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoRequest.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+import java.util.List;
+import java.util.Arrays;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
+import org.apache.hadoop.yarn.util.Records;
+
+@Private
+public abstract class ReportNewCollectorInfoRequest {
+
+ public static ReportNewCollectorInfoRequest newInstance(
+ List<AppCollectorsMap> appCollectorsList) {
+ ReportNewCollectorInfoRequest request =
+ Records.newRecord(ReportNewCollectorInfoRequest.class);
+ request.setAppCollectorsList(appCollectorsList);
+ return request;
+ }
+
+ public static ReportNewCollectorInfoRequest newInstance(
+ ApplicationId id, String collectorAddr) {
+ ReportNewCollectorInfoRequest request =
+ Records.newRecord(ReportNewCollectorInfoRequest.class);
+ request.setAppCollectorsList(
+ Arrays.asList(AppCollectorsMap.newInstance(id, collectorAddr)));
+ return request;
+ }
+
+ public abstract List<AppCollectorsMap> getAppCollectorsList();
+
+ public abstract void setAppCollectorsList(
+ List<AppCollectorsMap> appCollectorsList);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java
new file mode 100644
index 0000000..4157c47
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewCollectorInfoResponse.java
@@ -0,0 +1,32 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.util.Records;
+
+public abstract class ReportNewCollectorInfoResponse {
+
+ @Private
+ public static ReportNewCollectorInfoResponse newInstance() {
+ ReportNewCollectorInfoResponse response =
+ Records.newRecord(ReportNewCollectorInfoResponse.class);
+ return response;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index 02e0a3d..fa0cf5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -34,8 +34,8 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.LogAggregationReportProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatRequestProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeLabelsProto;
@@ -58,7 +58,7 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private Set<NodeLabel> labels = null;
private List<LogAggregationReport> logAggregationReportsForApps = null;
- Map<ApplicationId, String> registeredAggregators = null;
+ Map<ApplicationId, String> registeredCollectors = null;
public NodeHeartbeatRequestPBImpl() {
builder = NodeHeartbeatRequestProto.newBuilder();
@@ -114,8 +114,8 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
if (this.logAggregationReportsForApps != null) {
addLogAggregationStatusForAppsToProto();
}
- if (this.registeredAggregators != null) {
- addRegisteredAggregatorsToProto();
+ if (this.registeredCollectors != null) {
+ addRegisteredCollectorsToProto();
}
}
@@ -158,13 +158,13 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
return ((LogAggregationReportPBImpl) value).getProto();
}
- private void addRegisteredAggregatorsToProto() {
+ private void addRegisteredCollectorsToProto() {
maybeInitBuilder();
- builder.clearRegisteredAggregators();
- for (Map.Entry<ApplicationId, String> entry : registeredAggregators.entrySet()) {
- builder.addRegisteredAggregators(AppAggregatorsMapProto.newBuilder()
+ builder.clearRegisteredCollectors();
+ for (Map.Entry<ApplicationId, String> entry : registeredCollectors.entrySet()) {
+ builder.addRegisteredCollectors(AppCollectorsMapProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
- .setAppAggregatorAddr(entry.getValue()));
+ .setAppCollectorAddr(entry.getValue()));
}
}
@@ -248,35 +248,35 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
builder.clearLastKnownNmTokenMasterKey();
this.lastKnownNMTokenMasterKey = masterKey;
}
-
+
@Override
- public Map<ApplicationId, String> getRegisteredAggregators() {
- if (this.registeredAggregators != null) {
- return this.registeredAggregators;
+ public Map<ApplicationId, String> getRegisteredCollectors() {
+ if (this.registeredCollectors != null) {
+ return this.registeredCollectors;
}
- initRegisteredAggregators();
- return registeredAggregators;
+ initRegisteredCollectors();
+ return registeredCollectors;
}
-
- private void initRegisteredAggregators() {
+
+ private void initRegisteredCollectors() {
NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
- List<AppAggregatorsMapProto> list = p.getRegisteredAggregatorsList();
- this.registeredAggregators = new HashMap<ApplicationId, String> ();
- for (AppAggregatorsMapProto c : list) {
+ List<AppCollectorsMapProto> list = p.getRegisteredCollectorsList();
+ this.registeredCollectors = new HashMap<ApplicationId, String> ();
+ for (AppCollectorsMapProto c : list) {
ApplicationId appId = convertFromProtoFormat(c.getAppId());
- this.registeredAggregators.put(appId, c.getAppAggregatorAddr());
+ this.registeredCollectors.put(appId, c.getAppCollectorAddr());
}
}
-
+
@Override
- public void setRegisteredAggregators(
- Map<ApplicationId, String> registeredAggregators) {
- if (registeredAggregators == null || registeredAggregators.isEmpty()) {
+ public void setRegisteredCollectors(
+ Map<ApplicationId, String> registeredCollectors) {
+ if (registeredCollectors == null || registeredCollectors.isEmpty()) {
return;
}
maybeInitBuilder();
- this.registeredAggregators = new HashMap<ApplicationId, String>();
- this.registeredAggregators.putAll(registeredAggregators);
+ this.registeredCollectors = new HashMap<ApplicationId, String>();
+ this.registeredCollectors.putAll(registeredCollectors);
}
private NodeStatusPBImpl convertFromProtoFormat(NodeStatusProto p) {
@@ -286,11 +286,11 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private NodeStatusProto convertToProtoFormat(NodeStatus t) {
return ((NodeStatusPBImpl)t).getProto();
}
-
+
private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
return new ApplicationIdPBImpl(p);
}
-
+
private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
return ((ApplicationIdPBImpl) t).getProto();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 197245c..394e5ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -35,7 +35,7 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.SystemCredentialsForAppsProto;
@@ -56,8 +56,8 @@ public class NodeHeartbeatResponsePBImpl extends
private List<ContainerId> containersToBeRemovedFromNM = null;
private List<ApplicationId> applicationsToCleanup = null;
private Map<ApplicationId, ByteBuffer> systemCredentials = null;
-
- Map<ApplicationId, String> appAggregatorsMap = null;
+
+ Map<ApplicationId, String> appCollectorsMap = null;
private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null;
@@ -99,9 +99,9 @@ public class NodeHeartbeatResponsePBImpl extends
if (this.systemCredentials != null) {
addSystemCredentialsToProto();
}
-
- if (this.appAggregatorsMap != null) {
- addAppAggregatorsMapToProto();
+
+ if (this.appCollectorsMap != null) {
+ addAppCollectorsMapToProto();
}
}
@@ -115,14 +115,14 @@ public class NodeHeartbeatResponsePBImpl extends
entry.getValue().duplicate())));
}
}
-
- private void addAppAggregatorsMapToProto() {
+
+ private void addAppCollectorsMapToProto() {
maybeInitBuilder();
- builder.clearAppAggregatorsMap();
- for (Map.Entry<ApplicationId, String> entry : appAggregatorsMap.entrySet()) {
- builder.addAppAggregatorsMap(AppAggregatorsMapProto.newBuilder()
+ builder.clearAppCollectorsMap();
+ for (Map.Entry<ApplicationId, String> entry : appCollectorsMap.entrySet()) {
+ builder.addAppCollectorsMap(AppCollectorsMapProto.newBuilder()
.setAppId(convertToProtoFormat(entry.getKey()))
- .setAppAggregatorAddr(entry.getValue()));
+ .setAppCollectorAddr(entry.getValue()));
}
}
@@ -434,14 +434,14 @@ public class NodeHeartbeatResponsePBImpl extends
initSystemCredentials();
return systemCredentials;
}
-
+
@Override
- public Map<ApplicationId, String> getAppAggregatorsMap() {
- if (this.appAggregatorsMap != null) {
- return this.appAggregatorsMap;
+ public Map<ApplicationId, String> getAppCollectorsMap() {
+ if (this.appCollectorsMap != null) {
+ return this.appCollectorsMap;
}
- initAppAggregatorsMap();
- return appAggregatorsMap;
+ initAppCollectorsMap();
+ return appCollectorsMap;
}
private void initSystemCredentials() {
@@ -454,14 +454,14 @@ public class NodeHeartbeatResponsePBImpl extends
this.systemCredentials.put(appId, byteBuffer);
}
}
-
- private void initAppAggregatorsMap() {
+
+ private void initAppCollectorsMap() {
NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<AppAggregatorsMapProto> list = p.getAppAggregatorsMapList();
- this.appAggregatorsMap = new HashMap<ApplicationId, String> ();
- for (AppAggregatorsMapProto c : list) {
+ List<AppCollectorsMapProto> list = p.getAppCollectorsMapList();
+ this.appCollectorsMap = new HashMap<ApplicationId, String> ();
+ for (AppCollectorsMapProto c : list) {
ApplicationId appId = convertFromProtoFormat(c.getAppId());
- this.appAggregatorsMap.put(appId, c.getAppAggregatorAddr());
+ this.appCollectorsMap.put(appId, c.getAppCollectorAddr());
}
}
@@ -475,16 +475,16 @@ public class NodeHeartbeatResponsePBImpl extends
this.systemCredentials = new HashMap<ApplicationId, ByteBuffer>();
this.systemCredentials.putAll(systemCredentials);
}
-
+
@Override
- public void setAppAggregatorsMap(
- Map<ApplicationId, String> appAggregatorsMap) {
- if (appAggregatorsMap == null || appAggregatorsMap.isEmpty()) {
+ public void setAppCollectorsMap(
+ Map<ApplicationId, String> appCollectorsMap) {
+ if (appCollectorsMap == null || appCollectorsMap.isEmpty()) {
return;
}
maybeInitBuilder();
- this.appAggregatorsMap = new HashMap<ApplicationId, String>();
- this.appAggregatorsMap.putAll(appAggregatorsMap);
+ this.appCollectorsMap = new HashMap<ApplicationId, String>();
+ this.appCollectorsMap.putAll(appCollectorsMap);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java
deleted file mode 100644
index eb7beef..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoRequestPBImpl.java
+++ /dev/null
@@ -1,142 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
-
-import java.util.ArrayList;
-import java.util.List;
-
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProtoOrBuilder;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap;
-import org.apache.hadoop.yarn.server.api.records.impl.pb.AppAggregatorsMapPBImpl;
-
-public class ReportNewAggregatorsInfoRequestPBImpl extends
- ReportNewAggregatorsInfoRequest {
-
- ReportNewAggregatorsInfoRequestProto proto =
- ReportNewAggregatorsInfoRequestProto.getDefaultInstance();
-
- ReportNewAggregatorsInfoRequestProto.Builder builder = null;
- boolean viaProto = false;
-
- private List<AppAggregatorsMap> aggregatorsList = null;
-
- public ReportNewAggregatorsInfoRequestPBImpl() {
- builder = ReportNewAggregatorsInfoRequestProto.newBuilder();
- }
-
- public ReportNewAggregatorsInfoRequestPBImpl(
- ReportNewAggregatorsInfoRequestProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public ReportNewAggregatorsInfoRequestProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- private void mergeLocalToProto() {
- if (viaProto)
- maybeInitBuilder();
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void mergeLocalToBuilder() {
- if (aggregatorsList != null) {
- addLocalAggregatorsToProto();
- }
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ReportNewAggregatorsInfoRequestProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private void addLocalAggregatorsToProto() {
- maybeInitBuilder();
- builder.clearAppAggregators();
- List<AppAggregatorsMapProto> protoList =
- new ArrayList<AppAggregatorsMapProto>();
- for (AppAggregatorsMap m : this.aggregatorsList) {
- protoList.add(convertToProtoFormat(m));
- }
- builder.addAllAppAggregators(protoList);
- }
-
- private void initLocalAggregatorsList() {
- ReportNewAggregatorsInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
- List<AppAggregatorsMapProto> aggregatorsList =
- p.getAppAggregatorsList();
- this.aggregatorsList = new ArrayList<AppAggregatorsMap>();
- for (AppAggregatorsMapProto m : aggregatorsList) {
- this.aggregatorsList.add(convertFromProtoFormat(m));
- }
- }
-
- @Override
- public List<AppAggregatorsMap> getAppAggregatorsList() {
- if (this.aggregatorsList == null) {
- initLocalAggregatorsList();
- }
- return this.aggregatorsList;
- }
-
- @Override
- public void setAppAggregatorsList(List<AppAggregatorsMap> appAggregatorsList) {
- maybeInitBuilder();
- if (appAggregatorsList == null) {
- builder.clearAppAggregators();
- }
- this.aggregatorsList = appAggregatorsList;
- }
-
- private AppAggregatorsMapPBImpl convertFromProtoFormat(
- AppAggregatorsMapProto p) {
- return new AppAggregatorsMapPBImpl(p);
- }
-
- private AppAggregatorsMapProto convertToProtoFormat(
- AppAggregatorsMap m) {
- return ((AppAggregatorsMapPBImpl) m).getProto();
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java
deleted file mode 100644
index 0f0925a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewAggregatorsInfoResponsePBImpl.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoResponseProto;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse;
-
-import com.google.protobuf.TextFormat;
-
-@Private
-@Unstable
-public class ReportNewAggregatorsInfoResponsePBImpl extends
- ReportNewAggregatorsInfoResponse {
-
- ReportNewAggregatorsInfoResponseProto proto =
- ReportNewAggregatorsInfoResponseProto.getDefaultInstance();
-
- ReportNewAggregatorsInfoResponseProto.Builder builder = null;
-
- boolean viaProto = false;
-
- public ReportNewAggregatorsInfoResponsePBImpl() {
- builder = ReportNewAggregatorsInfoResponseProto.newBuilder();
- }
-
- public ReportNewAggregatorsInfoResponsePBImpl(ReportNewAggregatorsInfoResponseProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public ReportNewAggregatorsInfoResponseProto getProto() {
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java
new file mode 100644
index 0000000..5dd8f17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoRequestPBImpl.java
@@ -0,0 +1,142 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
+import org.apache.hadoop.yarn.server.api.records.impl.pb.AppCollectorsMapPBImpl;
+
+public class ReportNewCollectorInfoRequestPBImpl extends
+ ReportNewCollectorInfoRequest {
+
+ ReportNewCollectorInfoRequestProto proto =
+ ReportNewCollectorInfoRequestProto.getDefaultInstance();
+
+ ReportNewCollectorInfoRequestProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private List<AppCollectorsMap> collectorsList = null;
+
+ public ReportNewCollectorInfoRequestPBImpl() {
+ builder = ReportNewCollectorInfoRequestProto.newBuilder();
+ }
+
+ public ReportNewCollectorInfoRequestPBImpl(
+ ReportNewCollectorInfoRequestProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReportNewCollectorInfoRequestProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (collectorsList != null) {
+ addLocalCollectorsToProto();
+ }
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ReportNewCollectorInfoRequestProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void addLocalCollectorsToProto() {
+ maybeInitBuilder();
+ builder.clearAppCollectors();
+ List<AppCollectorsMapProto> protoList =
+ new ArrayList<AppCollectorsMapProto>();
+ for (AppCollectorsMap m : this.collectorsList) {
+ protoList.add(convertToProtoFormat(m));
+ }
+ builder.addAllAppCollectors(protoList);
+ }
+
+ private void initLocalCollectorsList() {
+ ReportNewCollectorInfoRequestProtoOrBuilder p = viaProto ? proto : builder;
+ List<AppCollectorsMapProto> collectorsList =
+ p.getAppCollectorsList();
+ this.collectorsList = new ArrayList<AppCollectorsMap>();
+ for (AppCollectorsMapProto m : collectorsList) {
+ this.collectorsList.add(convertFromProtoFormat(m));
+ }
+ }
+
+ @Override
+ public List<AppCollectorsMap> getAppCollectorsList() {
+ if (this.collectorsList == null) {
+ initLocalCollectorsList();
+ }
+ return this.collectorsList;
+ }
+
+ @Override
+ public void setAppCollectorsList(List<AppCollectorsMap> appCollectorsList) {
+ maybeInitBuilder();
+ if (appCollectorsList == null) {
+ builder.clearAppCollectors();
+ }
+ this.collectorsList = appCollectorsList;
+ }
+
+ private AppCollectorsMapPBImpl convertFromProtoFormat(
+ AppCollectorsMapProto p) {
+ return new AppCollectorsMapPBImpl(p);
+ }
+
+ private AppCollectorsMapProto convertToProtoFormat(
+ AppCollectorsMap m) {
+ return ((AppCollectorsMapPBImpl) m).getProto();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java
new file mode 100644
index 0000000..7c90675
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/ReportNewCollectorInfoResponsePBImpl.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class ReportNewCollectorInfoResponsePBImpl extends
+ ReportNewCollectorInfoResponse {
+
+ ReportNewCollectorInfoResponseProto proto =
+ ReportNewCollectorInfoResponseProto.getDefaultInstance();
+
+ ReportNewCollectorInfoResponseProto.Builder builder = null;
+
+ boolean viaProto = false;
+
+ public ReportNewCollectorInfoResponsePBImpl() {
+ builder = ReportNewCollectorInfoResponseProto.newBuilder();
+ }
+
+ public ReportNewCollectorInfoResponsePBImpl(ReportNewCollectorInfoResponseProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ReportNewCollectorInfoResponseProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java
deleted file mode 100644
index 67c377d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppAggregatorsMap.java
+++ /dev/null
@@ -1,33 +0,0 @@
-package org.apache.hadoop.yarn.server.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.util.Records;
-
-
-@Private
-public abstract class AppAggregatorsMap {
-
- public static AppAggregatorsMap newInstance(
- ApplicationId id, String aggregatorAddr) {
- AppAggregatorsMap appAggregatorMap =
- Records.newRecord(AppAggregatorsMap.class);
- appAggregatorMap.setApplicationId(id);
- appAggregatorMap.setAggregatorAddr(aggregatorAddr);
- return appAggregatorMap;
- }
-
- public abstract ApplicationId getApplicationId();
-
- public abstract void setApplicationId(
- ApplicationId id);
-
- public abstract String getAggregatorAddr();
-
- public abstract void setAggregatorAddr(
- String addr);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java
new file mode 100644
index 0000000..07e1d92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/AppCollectorsMap.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.util.Records;
+
+
+@Private
+public abstract class AppCollectorsMap {
+
+ public static AppCollectorsMap newInstance(
+ ApplicationId id, String collectorAddr) {
+ AppCollectorsMap appCollectorsMap =
+ Records.newRecord(AppCollectorsMap.class);
+ appCollectorsMap.setApplicationId(id);
+ appCollectorsMap.setCollectorAddr(collectorAddr);
+ return appCollectorsMap;
+ }
+
+ public abstract ApplicationId getApplicationId();
+
+ public abstract void setApplicationId(ApplicationId id);
+
+ public abstract String getCollectorAddr();
+
+ public abstract void setCollectorAddr(String addr);
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java
deleted file mode 100644
index 32903e2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppAggregatorsMapPBImpl.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-package org.apache.hadoop.yarn.server.api.records.impl.pb;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap;
-
-import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppAggregatorsMapProtoOrBuilder;
-
-import com.google.protobuf.TextFormat;
-
-@Private
-@Unstable
-public class AppAggregatorsMapPBImpl extends AppAggregatorsMap {
-
- AppAggregatorsMapProto proto =
- AppAggregatorsMapProto.getDefaultInstance();
-
- AppAggregatorsMapProto.Builder builder = null;
- boolean viaProto = false;
-
- private ApplicationId appId = null;
- private String aggregatorAddr = null;
-
- public AppAggregatorsMapPBImpl() {
- builder = AppAggregatorsMapProto.newBuilder();
- }
-
- public AppAggregatorsMapPBImpl(AppAggregatorsMapProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public AppAggregatorsMapProto getProto() {
- mergeLocalToProto();
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- @Override
- public int hashCode() {
- return getProto().hashCode();
- }
-
- @Override
- public boolean equals(Object other) {
- if (other == null)
- return false;
- if (other.getClass().isAssignableFrom(this.getClass())) {
- return this.getProto().equals(this.getClass().cast(other).getProto());
- }
- return false;
- }
-
- @Override
- public String toString() {
- return TextFormat.shortDebugString(getProto());
- }
-
- @Override
- public ApplicationId getApplicationId() {
- AppAggregatorsMapProtoOrBuilder p = viaProto ? proto : builder;
- if (this.appId == null && p.hasAppId()) {
- this.appId = convertFromProtoFormat(p.getAppId());
- }
- return this.appId;
- }
-
- @Override
- public String getAggregatorAddr() {
- AppAggregatorsMapProtoOrBuilder p = viaProto ? proto : builder;
- if (this.aggregatorAddr == null
- && p.hasAppAggregatorAddr()) {
- this.aggregatorAddr = p.getAppAggregatorAddr();
- }
- return this.aggregatorAddr;
- }
-
- @Override
- public void setApplicationId(ApplicationId appId) {
- maybeInitBuilder();
- if (appId == null) {
- builder.clearAppId();
- }
- this.appId = appId;
- }
-
- @Override
- public void setAggregatorAddr(String aggregatorAddr) {
- maybeInitBuilder();
- if (aggregatorAddr == null) {
- builder.clearAppAggregatorAddr();
- }
- this.aggregatorAddr = aggregatorAddr;
- }
-
- private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
- return new ApplicationIdPBImpl(p);
- }
-
- private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
- return ((ApplicationIdPBImpl) t).getProto();
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = AppAggregatorsMapProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- private void mergeLocalToProto() {
- if (viaProto) {
- maybeInitBuilder();
- }
- mergeLocalToBuilder();
- proto = builder.build();
- viaProto = true;
- }
-
- private void mergeLocalToBuilder() {
- if (this.appId != null) {
- builder.setAppId(convertToProtoFormat(this.appId));
- }
- if (this.aggregatorAddr != null) {
- builder.setAppAggregatorAddr(this.aggregatorAddr);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java
new file mode 100644
index 0000000..eb3bde4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/AppCollectorsMapPBImpl.java
@@ -0,0 +1,151 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
+import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
+
+import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.AppCollectorsMapProtoOrBuilder;
+
+import com.google.protobuf.TextFormat;
+
+@Private
+@Unstable
+public class AppCollectorsMapPBImpl extends AppCollectorsMap {
+
+ AppCollectorsMapProto proto =
+ AppCollectorsMapProto.getDefaultInstance();
+
+ AppCollectorsMapProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private ApplicationId appId = null;
+ private String collectorAddr = null;
+
+ public AppCollectorsMapPBImpl() {
+ builder = AppCollectorsMapProto.newBuilder();
+ }
+
+ public AppCollectorsMapPBImpl(AppCollectorsMapProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public AppCollectorsMapProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
+ @Override
+ public String toString() {
+ return TextFormat.shortDebugString(getProto());
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.appId == null && p.hasAppId()) {
+ this.appId = convertFromProtoFormat(p.getAppId());
+ }
+ return this.appId;
+ }
+
+ @Override
+ public String getCollectorAddr() {
+ AppCollectorsMapProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.collectorAddr == null
+ && p.hasAppCollectorAddr()) {
+ this.collectorAddr = p.getAppCollectorAddr();
+ }
+ return this.collectorAddr;
+ }
+
+ @Override
+ public void setApplicationId(ApplicationId appId) {
+ maybeInitBuilder();
+ if (appId == null) {
+ builder.clearAppId();
+ }
+ this.appId = appId;
+ }
+
+ @Override
+ public void setCollectorAddr(String collectorAddr) {
+ maybeInitBuilder();
+ if (collectorAddr == null) {
+ builder.clearAppCollectorAddr();
+ }
+ this.collectorAddr = collectorAddr;
+ }
+
+ private ApplicationIdPBImpl convertFromProtoFormat(ApplicationIdProto p) {
+ return new ApplicationIdPBImpl(p);
+ }
+
+ private ApplicationIdProto convertToProtoFormat(ApplicationId t) {
+ return ((ApplicationIdPBImpl) t).getProto();
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = AppCollectorsMapProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto) {
+ maybeInitBuilder();
+ }
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.appId != null) {
+ builder.setAppId(convertToProtoFormat(this.appId));
+ }
+ if (this.collectorAddr != null) {
+ builder.setAppCollectorAddr(this.collectorAddr);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto
deleted file mode 100644
index d7b05c1..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/aggregatornodemanager_protocol.proto
+++ /dev/null
@@ -1,29 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-option java_package = "org.apache.hadoop.yarn.proto";
-option java_outer_classname = "AggregatorNodemanagerProtocol";
-option java_generic_services = true;
-option java_generate_equals_and_hash = true;
-package hadoop.yarn;
-
-import "yarn_server_common_service_protos.proto";
-
-service AggregatorNodemanagerProtocolService {
- rpc reportNewAggregatorInfo (ReportNewAggregatorsInfoRequestProto) returns (ReportNewAggregatorsInfoResponseProto);
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
new file mode 100644
index 0000000..654a9f2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/collectornodemanager_protocol.proto
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+option java_package = "org.apache.hadoop.yarn.proto";
+option java_outer_classname = "CollectorNodemanagerProtocol";
+option java_generic_services = true;
+option java_generate_equals_and_hash = true;
+package hadoop.yarn;
+
+import "yarn_server_common_service_protos.proto";
+
+service CollectorNodemanagerProtocolService {
+ rpc reportNewCollectorInfo (ReportNewCollectorInfoRequestProto) returns (ReportNewCollectorInfoResponseProto);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 6a853f4..d358b83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -62,7 +62,7 @@ message NodeHeartbeatRequestProto {
optional MasterKeyProto last_known_nm_token_master_key = 3;
optional NodeLabelsProto nodeLabels = 4;
repeated LogAggregationReportProto log_aggregation_reports_for_apps = 5;
- repeated AppAggregatorsMapProto registered_aggregators = 6;
+ repeated AppCollectorsMapProto registered_collectors = 6;
}
message LogAggregationReportProto {
@@ -83,7 +83,7 @@ message NodeHeartbeatResponseProto {
repeated ContainerIdProto containers_to_be_removed_from_nm = 9;
repeated SystemCredentialsForAppsProto system_credentials_for_apps = 10;
optional bool areNodeLabelsAcceptedByRM = 11 [default = false];
- repeated AppAggregatorsMapProto app_aggregators_map = 12;
+ repeated AppCollectorsMapProto app_collectors_map = 12;
}
message SystemCredentialsForAppsProto {
@@ -92,21 +92,21 @@ message SystemCredentialsForAppsProto {
}
////////////////////////////////////////////////////////////////////////
-////// From aggregator_nodemanager_protocol ////////////////////////////
+////// From collector_nodemanager_protocol ////////////////////////////
////////////////////////////////////////////////////////////////////////
-message AppAggregatorsMapProto {
+message AppCollectorsMapProto {
optional ApplicationIdProto appId = 1;
- optional string appAggregatorAddr = 2;
+ optional string appCollectorAddr = 2;
}
//////////////////////////////////////////////////////
-/////// aggregator_nodemanager_protocol //////////////
+/////// collector_nodemanager_protocol //////////////
//////////////////////////////////////////////////////
-message ReportNewAggregatorsInfoRequestProto {
- repeated AppAggregatorsMapProto app_aggregators = 1;
+message ReportNewCollectorInfoRequestProto {
+ repeated AppCollectorsMapProto app_collectors = 1;
}
-message ReportNewAggregatorsInfoResponseProto {
+message ReportNewCollectorInfoResponseProto {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
index af9d60f..cfc3dc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
@@ -59,10 +59,10 @@ import org.apache.hadoop.yarn.ipc.HadoopYarnProtoRPC;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse;
-import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Test;
@@ -72,15 +72,15 @@ public class TestRPC {
private static final String EXCEPTION_MSG = "test error";
private static final String EXCEPTION_CAUSE = "exception cause";
private static final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
-
- public static final String ILLEGAL_NUMBER_MESSAGE =
- "aggregators' number in ReportNewAggregatorsInfoRequest is not ONE.";
-
- public static final String DEFAULT_AGGREGATOR_ADDR = "localhost:0";
-
- public static final ApplicationId DEFAULT_APP_ID =
+
+ public static final String ILLEGAL_NUMBER_MESSAGE =
+ "collectors' number in ReportNewCollectorInfoRequest is not ONE.";
+
+ public static final String DEFAULT_COLLECTOR_ADDR = "localhost:0";
+
+ public static final ApplicationId DEFAULT_APP_ID =
ApplicationId.newInstance(0, 0);
-
+
@Test
public void testUnknownCall() {
Configuration conf = new Configuration();
@@ -112,17 +112,17 @@ public class TestRPC {
server.stop();
}
}
-
+
@Test
- public void testRPCOnAggregatorNodeManagerProtocol() throws IOException {
+ public void testRPCOnCollectorNodeManagerProtocol() throws IOException {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.IPC_RPC_IMPL, HadoopYarnProtoRPC.class
.getName());
YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
- Server server = rpc.getServer(AggregatorNodemanagerProtocol.class,
- new DummyNMAggregatorService(), addr, conf, null, 1);
+ Server server = rpc.getServer(CollectorNodemanagerProtocol.class,
+ new DummyNMCollectorService(), addr, conf, null, 1);
server.start();
// Test unrelated protocol wouldn't get response
@@ -141,31 +141,31 @@ public class TestRPC {
} catch (Exception e) {
e.printStackTrace();
}
-
- // Test AggregatorNodemanagerProtocol get proper response
- AggregatorNodemanagerProtocol proxy = (AggregatorNodemanagerProtocol)rpc.getProxy(
- AggregatorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf);
- // Verify request with DEFAULT_APP_ID and DEFAULT_AGGREGATOR_ADDR get
+
+ // Test CollectorNodemanagerProtocol get proper response
+ CollectorNodemanagerProtocol proxy = (CollectorNodemanagerProtocol)rpc.getProxy(
+ CollectorNodemanagerProtocol.class, NetUtils.getConnectAddress(server), conf);
+ // Verify request with DEFAULT_APP_ID and DEFAULT_COLLECTOR_ADDR get
// normally response.
try {
- ReportNewAggregatorsInfoRequest request =
- ReportNewAggregatorsInfoRequest.newInstance(
- DEFAULT_APP_ID, DEFAULT_AGGREGATOR_ADDR);
- proxy.reportNewAggregatorInfo(request);
+ ReportNewCollectorInfoRequest request =
+ ReportNewCollectorInfoRequest.newInstance(
+ DEFAULT_APP_ID, DEFAULT_COLLECTOR_ADDR);
+ proxy.reportNewCollectorInfo(request);
} catch (YarnException e) {
Assert.fail("RPC call failured is not expected here.");
}
-
- // Verify empty request get YarnException back (by design in
- // DummyNMAggregatorService)
+
+ // Verify empty request get YarnException back (by design in
+ // DummyNMCollectorService)
try {
- proxy.reportNewAggregatorInfo(Records
- .newRecord(ReportNewAggregatorsInfoRequest.class));
+ proxy.reportNewCollectorInfo(Records
+ .newRecord(ReportNewCollectorInfoRequest.class));
Assert.fail("Excepted RPC call to fail with YarnException.");
} catch (YarnException e) {
Assert.assertTrue(e.getMessage().contains(ILLEGAL_NUMBER_MESSAGE));
}
-
+
server.stop();
}
@@ -173,21 +173,21 @@ public class TestRPC {
public void testHadoopProtoRPC() throws Exception {
test(HadoopYarnProtoRPC.class.getName());
}
-
+
private void test(String rpcClass) throws Exception {
Configuration conf = new Configuration();
conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
YarnRPC rpc = YarnRPC.create(conf);
String bindAddr = "localhost:0";
InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
- Server server = rpc.getServer(ContainerManagementProtocol.class,
+ Server server = rpc.getServer(ContainerManagementProtocol.class,
new DummyContainerManager(), addr, conf, null, 1);
server.start();
RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
- ContainerManagementProtocol proxy = (ContainerManagementProtocol)
- rpc.getProxy(ContainerManagementProtocol.class,
+ ContainerManagementProtocol proxy = (ContainerManagementProtocol)
+ rpc.getProxy(ContainerManagementProtocol.class,
NetUtils.getConnectAddress(server), conf);
- ContainerLaunchContext containerLaunchContext =
+ ContainerLaunchContext containerLaunchContext =
recordFactory.newRecordInstance(ContainerLaunchContext.class);
ApplicationId applicationId = ApplicationId.newInstance(0, 0);
@@ -251,7 +251,7 @@ public class TestRPC {
public GetContainerStatusesResponse getContainerStatuses(
GetContainerStatusesRequest request)
throws YarnException {
- GetContainerStatusesResponse response =
+ GetContainerStatusesResponse response =
recordFactory.newRecordInstance(GetContainerStatusesResponse.class);
response.setContainerStatuses(statuses);
return response;
@@ -283,9 +283,9 @@ public class TestRPC {
}
@Override
- public StopContainersResponse stopContainers(StopContainersRequest request)
+ public StopContainersResponse stopContainers(StopContainersRequest request)
throws YarnException {
- Exception e = new Exception(EXCEPTION_MSG,
+ Exception e = new Exception(EXCEPTION_MSG,
new Exception(EXCEPTION_CAUSE));
throw new YarnException(e);
}
@@ -314,32 +314,32 @@ public class TestRPC {
.buildTokenService(addr).toString());
return containerToken;
}
-
- // A dummy implementation for AggregatorNodemanagerProtocol for test purpose,
- // it only can accept one appID, aggregatorAddr pair or throw exceptions
- public class DummyNMAggregatorService
- implements AggregatorNodemanagerProtocol {
-
+
+ // A dummy implementation for CollectorNodemanagerProtocol for test purpose,
+ // it only can accept one appID, collectorAddr pair or throw exceptions
+ public class DummyNMCollectorService
+ implements CollectorNodemanagerProtocol {
+
@Override
- public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo(
- ReportNewAggregatorsInfoRequest request)
+ public ReportNewCollectorInfoResponse reportNewCollectorInfo(
+ ReportNewCollectorInfoRequest request)
throws YarnException, IOException {
- List<AppAggregatorsMap> appAggregators = request.getAppAggregatorsList();
- if (appAggregators.size() == 1) {
- // check default appID and aggregatorAddr
- AppAggregatorsMap appAggregator = appAggregators.get(0);
- Assert.assertEquals(appAggregator.getApplicationId(),
+ List<AppCollectorsMap> appCollectors = request.getAppCollectorsList();
+ if (appCollectors.size() == 1) {
+ // check default appID and collectorAddr
+ AppCollectorsMap appCollector = appCollectors.get(0);
+ Assert.assertEquals(appCollector.getApplicationId(),
DEFAULT_APP_ID);
- Assert.assertEquals(appAggregator.getAggregatorAddr(),
- DEFAULT_AGGREGATOR_ADDR);
+ Assert.assertEquals(appCollector.getCollectorAddr(),
+ DEFAULT_COLLECTOR_ADDR);
} else {
throw new YarnException(ILLEGAL_NUMBER_MESSAGE);
}
-
- ReportNewAggregatorsInfoResponse response =
- recordFactory.newRecordInstance(ReportNewAggregatorsInfoResponse.class);
+
+ ReportNewCollectorInfoResponse response =
+ recordFactory.newRecordInstance(ReportNewCollectorInfoResponse.class);
return response;
}
}
-
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index 876d1d4..d5e86f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -108,14 +108,14 @@ public class TestYarnServerApiClasses {
original.setLastKnownNMTokenMasterKey(getMasterKey());
original.setNodeStatus(getNodeStatus());
original.setNodeLabels(getValidNodeLabels());
- Map<ApplicationId, String> aggregators = getAggregators();
- original.setRegisteredAggregators(aggregators);
+ Map<ApplicationId, String> collectors = getCollectors();
+ original.setRegisteredCollectors(collectors);
NodeHeartbeatRequestPBImpl copy = new NodeHeartbeatRequestPBImpl(
original.getProto());
assertEquals(1, copy.getLastKnownContainerTokenMasterKey().getKeyId());
assertEquals(1, copy.getLastKnownNMTokenMasterKey().getKeyId());
assertEquals("localhost", copy.getNodeStatus().getNodeId().getHost());
- assertEquals(aggregators, copy.getRegisteredAggregators());
+ assertEquals(collectors, copy.getRegisteredCollectors());
// check labels are coming with valid values
Assert.assertTrue(original.getNodeLabels()
.containsAll(copy.getNodeLabels()));
@@ -152,8 +152,8 @@ public class TestYarnServerApiClasses {
original.setNextHeartBeatInterval(1000);
original.setNodeAction(NodeAction.NORMAL);
original.setResponseId(100);
- Map<ApplicationId, String> aggregators = getAggregators();
- original.setAppAggregatorsMap(aggregators);
+ Map<ApplicationId, String> collectors = getCollectors();
+ original.setAppCollectorsMap(collectors);
NodeHeartbeatResponsePBImpl copy = new NodeHeartbeatResponsePBImpl(
original.getProto());
@@ -163,7 +163,7 @@ public class TestYarnServerApiClasses {
assertEquals(1, copy.getContainerTokenMasterKey().getKeyId());
assertEquals(1, copy.getNMTokenMasterKey().getKeyId());
assertEquals("testDiagnosticMessage", copy.getDiagnosticsMessage());
- assertEquals(aggregators, copy.getAppAggregatorsMap());
+ assertEquals(collectors, copy.getAppCollectorsMap());
assertEquals(false, copy.getAreNodeLabelsAcceptedByRM());
}
@@ -323,15 +323,15 @@ public class TestYarnServerApiClasses {
return nodeLabels;
}
- private Map<ApplicationId, String> getAggregators() {
+ private Map<ApplicationId, String> getCollectors() {
ApplicationId appID = ApplicationId.newInstance(1L, 1);
- String aggregatorAddr = "localhost:0";
- Map<ApplicationId, String> aggregatorMap =
+ String collectorAddr = "localhost:0";
+ Map<ApplicationId, String> collectorMap =
new HashMap<ApplicationId, String>();
- aggregatorMap.put(appID, aggregatorAddr);
- return aggregatorMap;
+ collectorMap.put(appID, collectorAddr);
+ return collectorMap;
}
-
+
private ContainerStatus getContainerStatus(int applicationId,
int containerID, int appAttemptId) {
ContainerStatus status = recordFactory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index ad01a55..fed7113 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -59,19 +59,19 @@ public interface Context {
ConcurrentMap<ApplicationId, Application> getApplications();
Map<ApplicationId, Credentials> getSystemCredentialsForApps();
-
+
/**
- * Get the registered aggregators that located on this NM.
+ * Get the registered collectors that located on this NM.
* @return registered
*/
- Map<ApplicationId, String> getRegisteredAggregators();
-
+ Map<ApplicationId, String> getRegisteredCollectors();
+
/**
- * Return the known aggregators which get from RM for all active applications
+ * Return the known collectors which get from RM for all active applications
* running on this NM.
- * @return known aggregators.
+ * @return known collectors.
*/
- Map<ApplicationId, String> getKnownAggregators();
+ Map<ApplicationId, String> getKnownCollectors();
ConcurrentMap<ContainerId, Container> getContainers();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 6513179..441388b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
-import org.apache.hadoop.yarn.server.nodemanager.aggregatormanager.NMAggregatorService;
+import org.apache.hadoop.yarn.server.nodemanager.collectormanager.NMCollectorService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -94,7 +94,7 @@ public class NodeManager extends CompositeService
private Context context;
private AsyncDispatcher dispatcher;
private ContainerManagerImpl containerManager;
- private NMAggregatorService nmAggregatorService;
+ private NMCollectorService nmCollectorService;
private NodeStatusUpdater nodeStatusUpdater;
private NodeResourceMonitor nodeResourceMonitor;
private static CompositeServiceShutdownHook nodeManagerShutdownHook;
@@ -170,9 +170,9 @@ public class NodeManager extends CompositeService
return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
metrics, aclsManager, dirsHandler);
}
-
- protected NMAggregatorService createNMAggregatorService(Context context) {
- return new NMAggregatorService(context);
+
+ protected NMCollectorService createNMCollectorService(Context context) {
+ return new NMCollectorService(context);
}
protected WebServer createWebServer(Context nmContext,
@@ -349,9 +349,9 @@ public class NodeManager extends CompositeService
metrics.getJvmMetrics().setPauseMonitor(pauseMonitor);
DefaultMetricsSystem.initialize("NodeManager");
-
- this.nmAggregatorService = createNMAggregatorService(context);
- addService(nmAggregatorService);
+
+ this.nmCollectorService = createNMCollectorService(context);
+ addService(nmCollectorService);
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
@@ -447,11 +447,11 @@ public class NodeManager extends CompositeService
protected final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
-
- protected Map<ApplicationId, String> registeredAggregators =
+
+ protected Map<ApplicationId, String> registeredCollectors =
new ConcurrentHashMap<ApplicationId, String>();
-
- protected Map<ApplicationId, String> knownAggregators =
+
+ protected Map<ApplicationId, String> knownCollectors =
new ConcurrentHashMap<ApplicationId, String>();
private final NMContainerTokenSecretManager containerTokenSecretManager;
@@ -590,26 +590,26 @@ public class NodeManager extends CompositeService
}
@Override
- public Map<ApplicationId, String> getRegisteredAggregators() {
- return this.registeredAggregators;
+ public Map<ApplicationId, String> getRegisteredCollectors() {
+ return this.registeredCollectors;
}
- public void addRegisteredAggregators(
- Map<ApplicationId, String> newRegisteredAggregators) {
- this.registeredAggregators.putAll(newRegisteredAggregators);
- // Update to knownAggregators as well so it can immediately be consumed by
+ public void addRegisteredCollectors(
+ Map<ApplicationId, String> newRegisteredCollectors) {
+ this.registeredCollectors.putAll(newRegisteredCollectors);
+ // Update to knownCollectors as well so it can immediately be consumed by
// this NM's TimelineClient.
- this.knownAggregators.putAll(newRegisteredAggregators);
+ this.knownCollectors.putAll(newRegisteredCollectors);
}
-
+
@Override
- public Map<ApplicationId, String> getKnownAggregators() {
- return this.knownAggregators;
+ public Map<ApplicationId, String> getKnownCollectors() {
+ return this.knownCollectors;
}
- public void addKnownAggregators(
- Map<ApplicationId, String> knownAggregators) {
- this.knownAggregators.putAll(knownAggregators);
+ public void addKnownCollectors(
+ Map<ApplicationId, String> knownCollectors) {
+ this.knownCollectors.putAll(knownCollectors);
}
}
@@ -676,10 +676,10 @@ public class NodeManager extends CompositeService
public Context getNMContext() {
return this.context;
}
-
+
// For testing
- NMAggregatorService getNMAggregatorService() {
- return this.nmAggregatorService;
+ NMCollectorService getNMCollectorService() {
+ return this.nmCollectorService;
}
public static void main(String[] args) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 21d9ecf..c19a7d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -689,7 +689,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdaterImpl.this.context
.getNMTokenSecretManager().getCurrentKey(),
nodeLabelsForHeartbeat,
- NodeStatusUpdaterImpl.this.context.getRegisteredAggregators());
+ NodeStatusUpdaterImpl.this.context.getRegisteredCollectors());
if (logAggregationEnabled) {
// pull log aggregation status for application running in this NM
@@ -766,10 +766,11 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
((NMContext) context)
.setSystemCrendentialsForApps(parseCredentials(systemCredentials));
}
-
- Map<ApplicationId, String> knownAggregators = response.getAppAggregatorsMap();
- ((NodeManager.NMContext)context).addKnownAggregators(knownAggregators);
-
+
+ Map<ApplicationId, String> knownCollectors =
+ response.getAppCollectorsMap();
+ ((NodeManager.NMContext)context).addKnownCollectors(knownCollectors);
+
} catch (ConnectException e) {
//catch and throw the exception if tried MAX wait time to connect RM
dispatcher.getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java
deleted file mode 100644
index 17150ba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/aggregatormanager/NMAggregatorService.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.nodemanager.aggregatormanager;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.service.CompositeService;
-
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse;
-import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
-
-public class NMAggregatorService extends CompositeService implements
- AggregatorNodemanagerProtocol {
-
- private static final Log LOG = LogFactory.getLog(NMAggregatorService.class);
-
- final Context context;
-
- private Server server;
-
- public NMAggregatorService(Context context) {
-
- super(NMAggregatorService.class.getName());
- this.context = context;
- }
-
- @Override
- protected void serviceStart() throws Exception {
- Configuration conf = getConfig();
-
- InetSocketAddress aggregatorServerAddress = conf.getSocketAddr(
- YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_AGGREGATOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_PORT);
-
- Configuration serverConf = new Configuration(conf);
-
- // TODO Security settings.
- YarnRPC rpc = YarnRPC.create(conf);
-
- server =
- rpc.getServer(AggregatorNodemanagerProtocol.class, this,
- aggregatorServerAddress, serverConf,
- this.context.getNMTokenSecretManager(),
- conf.getInt(YarnConfiguration.NM_AGGREGATOR_SERVICE_THREAD_COUNT,
- YarnConfiguration.DEFAULT_NM_AGGREGATOR_SERVICE_THREAD_COUNT));
-
- server.start();
- // start remaining services
- super.serviceStart();
- LOG.info("NMAggregatorService started at " + aggregatorServerAddress);
- }
-
-
- @Override
- public void serviceStop() throws Exception {
- if (server != null) {
- server.stop();
- }
- // TODO may cleanup app aggregators running on this NM in future.
- super.serviceStop();
- }
-
- @Override
- public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo(
- ReportNewAggregatorsInfoRequest request) throws IOException {
- List<AppAggregatorsMap> newAggregatorsList = request.getAppAggregatorsList();
- if (newAggregatorsList != null && !newAggregatorsList.isEmpty()) {
- Map<ApplicationId, String> newAggregatorsMap =
- new HashMap<ApplicationId, String>();
- for (AppAggregatorsMap aggregator : newAggregatorsList) {
- newAggregatorsMap.put(aggregator.getApplicationId(), aggregator.getAggregatorAddr());
- }
- ((NodeManager.NMContext)context).addRegisteredAggregators(newAggregatorsMap);
- }
-
- return ReportNewAggregatorsInfoResponse.newInstance();
- }
-
-}
[17/50] [abbrv] hadoop git commit: MAPREDUCE-6327. Made MR AM use
timeline service v2 API to write history events and counters. Contributed by
Junping Du.
Posted by sj...@apache.org.
MAPREDUCE-6327. Made MR AM use timeline service v2 API to write history events and counters. Contributed by Junping Du.
(cherry picked from commit 5eeb2b156f8e108205945f0a1d06873cb51c3527)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e58f943
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e58f943
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e58f943
Branch: refs/heads/YARN-2928
Commit: 8e58f943ffbb9b6c02422984ee2c77ae45948c9e
Parents: 8fd1aa1
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue Apr 21 16:31:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:10 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 15 ++
.../jobhistory/JobHistoryEventHandler.java | 258 ++++++++++++++++---
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 23 ++
.../v2/app/rm/RMContainerAllocator.java | 9 +
.../hadoop/mapreduce/jobhistory/TestEvents.java | 8 +-
.../jobhistory/TestJobHistoryEventHandler.java | 9 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 5 +
.../mapreduce/jobhistory/AMStartedEvent.java | 18 ++
.../mapreduce/jobhistory/HistoryEvent.java | 4 +
.../mapreduce/jobhistory/JobFinishedEvent.java | 25 ++
.../jobhistory/JobInfoChangeEvent.java | 11 +
.../mapreduce/jobhistory/JobInitedEvent.java | 14 +
.../jobhistory/JobPriorityChangeEvent.java | 10 +
.../jobhistory/JobQueueChangeEvent.java | 10 +
.../jobhistory/JobStatusChangedEvent.java | 10 +
.../mapreduce/jobhistory/JobSubmittedEvent.java | 23 ++
.../JobUnsuccessfulCompletionEvent.java | 16 ++
.../jobhistory/MapAttemptFinishedEvent.java | 24 +-
.../jobhistory/NormalizedResourceEvent.java | 11 +
.../jobhistory/ReduceAttemptFinishedEvent.java | 25 +-
.../jobhistory/TaskAttemptFinishedEvent.java | 19 ++
.../jobhistory/TaskAttemptStartedEvent.java | 18 ++
.../TaskAttemptUnsuccessfulCompletionEvent.java | 24 ++
.../mapreduce/jobhistory/TaskFailedEvent.java | 19 ++
.../mapreduce/jobhistory/TaskFinishedEvent.java | 18 ++
.../mapreduce/jobhistory/TaskStartedEvent.java | 12 +
.../mapreduce/jobhistory/TaskUpdatedEvent.java | 10 +
.../mapreduce/util/JobHistoryEventUtils.java | 51 ++++
.../src/main/resources/mapred-default.xml | 7 +
.../mapred/TestMRTimelineEventHandling.java | 163 +++++++++++-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 21 +-
31 files changed, 838 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 305b29e..5ac0d3b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1,5 +1,20 @@
Hadoop MapReduce Change Log
+Branch YARN-2928: Timeline Server Next Generation: Phase 1
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ MAPREDUCE-6327. Made MR AM use timeline service v2 API to write history
+ events and counters. (Junping Du via zjshen)
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Trunk (Unreleased)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index b0bcfcd..1e84e44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -49,11 +52,13 @@ import org.apache.hadoop.mapreduce.JobCounter;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
+import org.apache.hadoop.mapreduce.v2.app.MRAppMaster;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
@@ -68,10 +73,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.codehaus.jackson.JsonNode;
-import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.node.ArrayNode;
-import org.codehaus.jackson.node.ObjectNode;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.common.annotations.VisibleForTesting;
/**
@@ -119,14 +122,24 @@ public class JobHistoryEventHandler extends AbstractService
protected static final Map<JobId, MetaInfo> fileMap =
Collections.<JobId,MetaInfo>synchronizedMap(new HashMap<JobId,MetaInfo>());
+
+ // For posting entities in new timeline service in a non-blocking way
+ // TODO YARN-3367 replace with event loop in TimelineClient.
+ private static ExecutorService threadPool =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
// should job completion be force when the AM shuts down?
protected volatile boolean forceJobCompletion = false;
protected TimelineClient timelineClient;
+
+ private boolean newTimelineServiceEnabled = false;
private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB";
private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK";
+ private static String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE = "MAPREDUCE_TASK_ATTEMPT";
public JobHistoryEventHandler(AppContext context, int startCount) {
super("JobHistoryEventHandler");
@@ -246,13 +259,22 @@ public class JobHistoryEventHandler extends AbstractService
MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD,
MRJobConfig.DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD);
+ // TODO replace MR specific configurations on timeline service with getting
+ // configuration from RM through registerApplicationMaster() in
+ // ApplicationMasterProtocol with return value for timeline service
+ // configuration status: off, on_with_v1 or on_with_v2.
if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
- timelineClient = TimelineClient.createTimelineClient();
+
+ timelineClient =
+ ((MRAppMaster.RunningAppContext)context).getTimelineClient();
timelineClient.init(conf);
- LOG.info("Timeline service is enabled");
+ newTimelineServiceEnabled = conf.getBoolean(
+ MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED,
+ MRJobConfig.DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED);
+ LOG.info("Timeline service is enabled: " + (newTimelineServiceEnabled? "v2" : "v1"));
LOG.info("Emitting job history data to the timeline server is enabled");
} else {
LOG.info("Timeline service is not enabled");
@@ -426,9 +448,26 @@ public class JobHistoryEventHandler extends AbstractService
if (timelineClient != null) {
timelineClient.stop();
}
+ shutdownAndAwaitTermination();
LOG.info("Stopped JobHistoryEventHandler. super.stop()");
super.serviceStop();
}
+
+ // TODO remove threadPool after adding non-blocking call in TimelineClient
+ private static void shutdownAndAwaitTermination() {
+ threadPool.shutdown();
+ try {
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
+ threadPool.shutdownNow();
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
+ LOG.error("ThreadPool did not terminate");
+ }
+ } catch (InterruptedException ie) {
+ threadPool.shutdownNow();
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ }
protected EventWriter createEventWriter(Path historyFilePath)
throws IOException {
@@ -583,8 +622,13 @@ public class JobHistoryEventHandler extends AbstractService
processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(),
event.getJobID());
if (timelineClient != null) {
- processEventForTimelineServer(historyEvent, event.getJobID(),
- event.getTimestamp());
+ if (newTimelineServiceEnabled) {
+ processEventForNewTimelineService(historyEvent, event.getJobID(),
+ event.getTimestamp());
+ } else {
+ processEventForTimelineServer(historyEvent, event.getJobID(),
+ event.getTimestamp());
+ }
}
if (LOG.isDebugEnabled()) {
LOG.debug("In HistoryEventHandler "
@@ -825,11 +869,11 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("FINISHED_MAPS", jfe.getFinishedMaps());
tEvent.addEventInfo("FINISHED_REDUCES", jfe.getFinishedReduces());
tEvent.addEventInfo("MAP_COUNTERS_GROUPS",
- countersToJSON(jfe.getTotalCounters()));
+ JobHistoryEventUtils.countersToJSON(jfe.getTotalCounters()));
tEvent.addEventInfo("REDUCE_COUNTERS_GROUPS",
- countersToJSON(jfe.getReduceCounters()));
+ JobHistoryEventUtils.countersToJSON(jfe.getReduceCounters()));
tEvent.addEventInfo("TOTAL_COUNTERS_GROUPS",
- countersToJSON(jfe.getTotalCounters()));
+ JobHistoryEventUtils.countersToJSON(jfe.getTotalCounters()));
tEvent.addEventInfo("JOB_STATUS", JobState.SUCCEEDED.toString());
tEntity.addEvent(tEvent);
tEntity.setEntityId(jobId.toString());
@@ -855,7 +899,7 @@ public class JobHistoryEventHandler extends AbstractService
tfe.getFailedAttemptID() == null ?
"" : tfe.getFailedAttemptID().toString());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tfe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tfe.getCounters()));
tEntity.addEvent(tEvent);
tEntity.setEntityId(tfe.getTaskId().toString());
tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
@@ -873,7 +917,7 @@ public class JobHistoryEventHandler extends AbstractService
TaskFinishedEvent tfe2 = (TaskFinishedEvent) event;
tEvent.addEventInfo("TASK_TYPE", tfe2.getTaskType().toString());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tfe2.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tfe2.getCounters()));
tEvent.addEventInfo("FINISH_TIME", tfe2.getFinishTime());
tEvent.addEventInfo("STATUS", TaskStatus.State.SUCCEEDED.toString());
tEvent.addEventInfo("SUCCESSFUL_TASK_ATTEMPT_ID",
@@ -895,7 +939,6 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("START_TIME", tase.getStartTime());
tEvent.addEventInfo("HTTP_PORT", tase.getHttpPort());
tEvent.addEventInfo("TRACKER_NAME", tase.getTrackerName());
- tEvent.addEventInfo("TASK_TYPE", tase.getTaskType().toString());
tEvent.addEventInfo("SHUFFLE_PORT", tase.getShufflePort());
tEvent.addEventInfo("CONTAINER_ID", tase.getContainerId() == null ?
"" : tase.getContainerId().toString());
@@ -928,7 +971,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("SORT_FINISH_TIME", tauce.getFinishTime());
tEvent.addEventInfo("MAP_FINISH_TIME", tauce.getFinishTime());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tauce.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tauce.getCounters()));
tEntity.addEvent(tEvent);
tEntity.setEntityId(tauce.getTaskId().toString());
tEntity.setEntityType(MAPREDUCE_TASK_ENTITY_TYPE);
@@ -942,7 +985,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("STATE", mafe.getState());
tEvent.addEventInfo("MAP_FINISH_TIME", mafe.getMapFinishTime());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(mafe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(mafe.getCounters()));
tEvent.addEventInfo("HOSTNAME", mafe.getHostname());
tEvent.addEventInfo("PORT", mafe.getPort());
tEvent.addEventInfo("RACK_NAME", mafe.getRackName());
@@ -964,7 +1007,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("SHUFFLE_FINISH_TIME", rafe.getShuffleFinishTime());
tEvent.addEventInfo("SORT_FINISH_TIME", rafe.getSortFinishTime());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(rafe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(rafe.getCounters()));
tEvent.addEventInfo("HOSTNAME", rafe.getHostname());
tEvent.addEventInfo("PORT", rafe.getPort());
tEvent.addEventInfo("RACK_NAME", rafe.getRackName());
@@ -983,7 +1026,7 @@ public class JobHistoryEventHandler extends AbstractService
tEvent.addEventInfo("STATUS", tafe.getTaskStatus());
tEvent.addEventInfo("STATE", tafe.getState());
tEvent.addEventInfo("COUNTERS_GROUPS",
- countersToJSON(tafe.getCounters()));
+ JobHistoryEventUtils.countersToJSON(tafe.getCounters()));
tEvent.addEventInfo("HOSTNAME", tafe.getHostname());
tEntity.addEvent(tEvent);
tEntity.setEntityId(tafe.getTaskId().toString());
@@ -1010,37 +1053,172 @@ public class JobHistoryEventHandler extends AbstractService
default:
break;
}
-
+
try {
timelineClient.putEntities(tEntity);
- } catch (IOException ex) {
- LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline"
- + "Server", ex);
- } catch (YarnException ex) {
+ } catch (IOException|YarnException ex) {
LOG.error("Error putting entity " + tEntity.getEntityId() + " to Timeline"
+ "Server", ex);
}
}
-
- @Private
- public JsonNode countersToJSON(Counters counters) {
- ObjectMapper mapper = new ObjectMapper();
- ArrayNode nodes = mapper.createArrayNode();
- if (counters != null) {
- for (CounterGroup counterGroup : counters) {
- ObjectNode groupNode = nodes.addObject();
- groupNode.put("NAME", counterGroup.getName());
- groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
- ArrayNode countersNode = groupNode.putArray("COUNTERS");
- for (Counter counter : counterGroup) {
- ObjectNode counterNode = countersNode.addObject();
- counterNode.put("NAME", counter.getName());
- counterNode.put("DISPLAY_NAME", counter.getDisplayName());
- counterNode.put("VALUE", counter.getValue());
+
+ private void putEntityWithoutBlocking(final TimelineClient timelineClient,
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity) {
+ Runnable publishWrapper = new Runnable() {
+ public void run() {
+ try {
+ timelineClient.putEntities(entity);
+ } catch (IOException|YarnException e) {
+ LOG.error("putEntityNonBlocking get failed: " + e);
+ throw new RuntimeException(e.toString());
}
}
+ };
+ threadPool.execute(publishWrapper);
+ }
+
+ // create JobEntity from HistoryEvent with adding other info, like:
+ // jobId, timestamp and entityType.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createJobEntity(HistoryEvent event, long timestamp, JobId jobId,
+ String entityType) {
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType);
+ entity.setId(jobId.toString());
+ return entity;
+ }
+
+ // create BaseEntity from HistoryEvent with adding other info, like:
+ // timestamp and entityType.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createBaseEntity(HistoryEvent event, long timestamp, String entityType) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent =
+ event.toTimelineEvent();
+ tEvent.setTimestamp(timestamp);
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
+ entity.addEvent(tEvent);
+ entity.setType(entityType);
+ return entity;
+ }
+
+ // create TaskEntity from HistoryEvent with adding other info, like:
+ // taskId, jobId, timestamp, entityType and relatedJobEntity.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createTaskEntity(HistoryEvent event, long timestamp, String taskId,
+ String entityType, String relatedJobEntity, JobId jobId) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType);
+ entity.setId(taskId);
+ entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
+ return entity;
+ }
+
+ // create TaskAttemptEntity from HistoryEvent with adding other info, like:
+ // timestamp, taskAttemptId, entityType, relatedTaskEntity and taskId.
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createTaskAttemptEntity(HistoryEvent event, long timestamp,
+ String taskAttemptId, String entityType, String relatedTaskEntity,
+ String taskId) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType);
+ entity.setId(taskAttemptId);
+ entity.addIsRelatedToEntity(relatedTaskEntity, taskId);
+ return entity;
+ }
+
+ private void processEventForNewTimelineService(HistoryEvent event, JobId jobId,
+ long timestamp) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null;
+ String taskId = null;
+ String taskAttemptId = null;
+
+ switch (event.getEventType()) {
+ // Handle job events
+ case JOB_SUBMITTED:
+ case JOB_STATUS_CHANGED:
+ case JOB_INFO_CHANGED:
+ case JOB_INITED:
+ case JOB_PRIORITY_CHANGED:
+ case JOB_QUEUE_CHANGED:
+ case JOB_FAILED:
+ case JOB_KILLED:
+ case JOB_ERROR:
+ case JOB_FINISHED:
+ case AM_STARTED:
+ case NORMALIZED_RESOURCE:
+ break;
+ // Handle task events
+ case TASK_STARTED:
+ taskId = ((TaskStartedEvent)event).getTaskId().toString();
+ break;
+ case TASK_FAILED:
+ taskId = ((TaskFailedEvent)event).getTaskId().toString();
+ break;
+ case TASK_UPDATED:
+ taskId = ((TaskUpdatedEvent)event).getTaskId().toString();
+ break;
+ case TASK_FINISHED:
+ taskId = ((TaskFinishedEvent)event).getTaskId().toString();
+ break;
+ case MAP_ATTEMPT_STARTED:
+ case CLEANUP_ATTEMPT_STARTED:
+ case REDUCE_ATTEMPT_STARTED:
+ case SETUP_ATTEMPT_STARTED:
+ taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptStartedEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case MAP_ATTEMPT_FAILED:
+ case CLEANUP_ATTEMPT_FAILED:
+ case REDUCE_ATTEMPT_FAILED:
+ case SETUP_ATTEMPT_FAILED:
+ case MAP_ATTEMPT_KILLED:
+ case CLEANUP_ATTEMPT_KILLED:
+ case REDUCE_ATTEMPT_KILLED:
+ case SETUP_ATTEMPT_KILLED:
+ taskId = ((TaskAttemptUnsuccessfulCompletionEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case MAP_ATTEMPT_FINISHED:
+ taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((MapAttemptFinishedEvent)event).getAttemptId().toString();
+ break;
+ case REDUCE_ATTEMPT_FINISHED:
+ taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((ReduceAttemptFinishedEvent)event).getAttemptId().toString();
+ break;
+ case SETUP_ATTEMPT_FINISHED:
+ case CLEANUP_ATTEMPT_FINISHED:
+ taskId = ((TaskAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptFinishedEvent)event).getAttemptId().toString();
+ break;
+ default:
+ LOG.warn("EventType: " + event.getEventType() + " cannot be recognized" +
+ " and handled by timeline service.");
+ return;
}
- return nodes;
+ if (taskId == null) {
+ // JobEntity
+ tEntity = createJobEntity(event, timestamp, jobId,
+ MAPREDUCE_JOB_ENTITY_TYPE);
+ } else {
+ if (taskAttemptId == null) {
+ // TaskEntity
+ tEntity = createTaskEntity(event, timestamp, taskId,
+ MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, jobId);
+ } else {
+ // TaskAttemptEntity
+ tEntity = createTaskAttemptEntity(event, timestamp, taskAttemptId,
+ MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE, MAPREDUCE_TASK_ENTITY_TYPE,
+ taskId);
+ }
+ }
+
+ putEntityWithoutBlocking(timelineClient, tEntity);
}
private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 6dc830f..dafb6e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -135,6 +135,7 @@ import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -1004,6 +1005,7 @@ public class MRAppMaster extends CompositeService {
private final Configuration conf;
private final ClusterInfo clusterInfo = new ClusterInfo();
private final ClientToAMTokenSecretManager clientToAMTokenSecretManager;
+ private TimelineClient timelineClient = null;
private final TaskAttemptFinishingMonitor taskAttemptFinishingMonitor;
@@ -1013,6 +1015,23 @@ public class MRAppMaster extends CompositeService {
this.clientToAMTokenSecretManager =
new ClientToAMTokenSecretManager(appAttemptID, null);
this.taskAttemptFinishingMonitor = taskAttemptFinishingMonitor;
+ if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
+ MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)
+ && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+
+ boolean newTimelineServiceEnabled = conf.getBoolean(
+ MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED,
+ MRJobConfig.DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED);
+
+ if (newTimelineServiceEnabled) {
+ // create new version TimelineClient
+ timelineClient = TimelineClient.createTimelineClient(
+ appAttemptID.getApplicationId());
+ } else {
+ timelineClient = TimelineClient.createTimelineClient();
+ }
+ }
}
@Override
@@ -1103,6 +1122,10 @@ public class MRAppMaster extends CompositeService {
return taskAttemptFinishingMonitor;
}
+ // Get Timeline Collector's address (get sync from RM)
+ public TimelineClient getTimelineClient() {
+ return timelineClient;
+ }
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
index ac4c586..e5338b5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
@@ -48,6 +48,7 @@ import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.MRAppMaster;
import org.apache.hadoop.mapreduce.v2.app.client.ClientService;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobCounterUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobDiagnosticsUpdateEvent;
@@ -769,6 +770,14 @@ public class RMContainerAllocator extends RMContainerRequestor
computeIgnoreBlacklisting();
handleUpdatedNodes(response);
+ String collectorAddr = response.getCollectorAddr();
+ MRAppMaster.RunningAppContext appContext =
+ (MRAppMaster.RunningAppContext)this.getContext();
+ if (collectorAddr != null && !collectorAddr.isEmpty()
+ && appContext.getTimelineClient() != null) {
+ appContext.getTimelineClient().setTimelineServiceAddress(
+ response.getCollectorAddr());
+ }
for (ContainerStatus cont : finishedContainers) {
LOG.info("Received completed container " + cont.getContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 7612ceb..e7d5006 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.junit.Test;
public class TestEvents {
@@ -404,7 +405,12 @@ public class TestEvents {
public void setDatum(Object datum) {
this.datum = datum;
}
-
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ return null;
+ }
+
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 2b07efb..2c159f1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
@@ -666,7 +667,7 @@ public class TestJobHistoryEventHandler {
group2.addCounter("MARTHA_JONES", "Martha Jones", 3);
group2.addCounter("DONNA_NOBLE", "Donna Noble", 2);
group2.addCounter("ROSE_TYLER", "Rose Tyler", 1);
- JsonNode jsonNode = jheh.countersToJSON(counters);
+ JsonNode jsonNode = JobHistoryEventUtils.countersToJSON(counters);
String jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
String expected = "[{\"NAME\":\"COMPANIONS\",\"DISPLAY_NAME\":\"Companions "
+ "of the Doctor\",\"COUNTERS\":[{\"NAME\":\"AMY_POND\",\"DISPLAY_NAME\""
@@ -689,19 +690,19 @@ public class TestJobHistoryEventHandler {
public void testCountersToJSONEmpty() throws Exception {
JobHistoryEventHandler jheh = new JobHistoryEventHandler(null, 0);
Counters counters = null;
- JsonNode jsonNode = jheh.countersToJSON(counters);
+ JsonNode jsonNode = JobHistoryEventUtils.countersToJSON(counters);
String jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
String expected = "[]";
Assert.assertEquals(expected, jsonStr);
counters = new Counters();
- jsonNode = jheh.countersToJSON(counters);
+ jsonNode = JobHistoryEventUtils.countersToJSON(counters);
jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
expected = "[]";
Assert.assertEquals(expected, jsonStr);
counters.addGroup("DOCTORS", "Incarnations of the Doctor");
- jsonNode = jheh.countersToJSON(counters);
+ jsonNode = JobHistoryEventUtils.countersToJSON(counters);
jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
expected = "[{\"NAME\":\"DOCTORS\",\"DISPLAY_NAME\":\"Incarnations of the "
+ "Doctor\",\"COUNTERS\":[]}]";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 59b887d..9c1b835 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -462,6 +462,11 @@ public interface MRJobConfig {
"mapreduce.job.emit-timeline-data";
public static final boolean DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA =
false;
+
+ public static final String MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED =
+ "mapreduce.job.new-timeline-service.enabled";
+ public static final boolean DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED =
+ false;
public static final String MR_PREFIX = "yarn.app.mapreduce.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
index ea2ca9e..bbc7090 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
@@ -20,8 +20,10 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.avro.util.Utf8;
@@ -166,4 +168,20 @@ public class AMStartedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.AM_STARTED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("APPLICATION_ATTEMPT_ID",
+ getAppAttemptId() == null ? "" : getAppAttemptId().toString());
+ tEvent.addInfo("CONTAINER_ID", getContainerId() == null ?
+ "" : getContainerId().toString());
+ tEvent.addInfo("NODE_MANAGER_HOST", getNodeManagerHost());
+ tEvent.addInfo("NODE_MANAGER_PORT", getNodeManagerPort());
+ tEvent.addInfo("NODE_MANAGER_HTTP_PORT", getNodeManagerHttpPort());
+ tEvent.addInfo("START_TIME", getStartTime());
+ return tEvent;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
index a30748c..61ce217 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Interface for event wrapper classes. Implementations each wrap an
@@ -37,4 +38,7 @@ public interface HistoryEvent {
/** Set the Avro datum wrapped by this. */
void setDatum(Object datum);
+
+ /** Map HistoryEvent to TimelineEvent */
+ TimelineEvent toTimelineEvent();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
index 0fa5868..80d3ee6 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
@@ -23,6 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful completion of job
@@ -133,4 +136,26 @@ public class JobFinishedEvent implements HistoryEvent {
public Counters getReduceCounters() {
return reduceCounters;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("NUM_MAPS", getFinishedMaps());
+ tEvent.addInfo("NUM_REDUCES", getFinishedReduces());
+ tEvent.addInfo("FAILED_MAPS", getFailedMaps());
+ tEvent.addInfo("FAILED_REDUCES", getFailedReduces());
+ tEvent.addInfo("FINISHED_MAPS", getFinishedMaps());
+ tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces());
+ tEvent.addInfo("MAP_COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getMapCounters()));
+ tEvent.addInfo("REDUCE_COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getReduceCounters()));
+ tEvent.addInfo("TOTAL_COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getTotalCounters()));
+ // TODO replace SUCCEEDED with JobState.SUCCEEDED.toString()
+ tEvent.addInfo("JOB_STATUS", "SUCCEEDED");
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
index b45f373..ad82443 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -64,5 +66,14 @@ public class JobInfoChangeEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.JOB_INFO_CHANGED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("SUBMIT_TIME", getSubmitTime());
+ tEvent.addInfo("LAUNCH_TIME", getLaunchTime());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
index 5abb40e..3e0f2f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -73,4 +75,16 @@ public class JobInitedEvent implements HistoryEvent {
}
/** Get whether the job's map and reduce stages were combined */
public boolean getUberized() { return datum.getUberized(); }
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("START_TIME", getLaunchTime());
+ tEvent.addInfo("STATUS", getStatus());
+ tEvent.addInfo("TOTAL_MAPS", getTotalMaps());
+ tEvent.addInfo("TOTAL_REDUCES", getTotalReduces());
+ tEvent.addInfo("UBERIZED", getUberized());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
index 6c32462..5deea0a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobPriority;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -64,5 +66,13 @@ public class JobPriorityChangeEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.JOB_PRIORITY_CHANGED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("PRIORITY", getPriority().toString());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
index 86078e6..b9dd359 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
@SuppressWarnings("deprecation")
public class JobQueueChangeEvent implements HistoryEvent {
@@ -59,5 +61,13 @@ public class JobQueueChangeEvent implements HistoryEvent {
}
return null;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("QUEUE_NAMES", getJobQueueName());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
index 9e11a55..a4f2da2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -60,5 +62,13 @@ public class JobStatusChangedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.JOB_STATUS_CHANGED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("STATUS", getStatus());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
index 845f6f7..47b2840 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
@@ -27,6 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -205,5 +207,26 @@ public class JobSubmittedEvent implements HistoryEvent {
}
/** Get the event type */
public EventType getEventType() { return EventType.JOB_SUBMITTED; }
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("SUBMIT_TIME", getSubmitTime());
+ tEvent.addInfo("QUEUE_NAME", getJobQueueName());
+ tEvent.addInfo("JOB_NAME", getJobName());
+ tEvent.addInfo("USER_NAME", getUserName());
+ tEvent.addInfo("JOB_CONF_PATH", getJobConfPath());
+ tEvent.addInfo("ACLS", getJobAcls());
+ tEvent.addInfo("JOB_QUEUE_NAME", getJobQueueName());
+ tEvent.addInfo("WORKLFOW_ID", getWorkflowId());
+ tEvent.addInfo("WORKFLOW_NAME", getWorkflowName());
+ tEvent.addInfo("WORKFLOW_NODE_NAME", getWorkflowNodeName());
+ tEvent.addInfo("WORKFLOW_ADJACENCIES",
+ getWorkflowAdjacencies());
+ tEvent.addInfo("WORKFLOW_TAGS", getWorkflowTags());
+
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
index 1b7773d..ea9798c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
@@ -24,6 +24,8 @@ import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import java.util.Collections;
@@ -119,4 +121,18 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent {
final CharSequence diagnostics = datum.getDiagnostics();
return diagnostics == null ? NODIAGS : diagnostics.toString();
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("NUM_MAPS", getFinishedMaps());
+ tEvent.addInfo("NUM_REDUCES", getFinishedReduces());
+ tEvent.addInfo("JOB_STATUS", getStatus());
+ tEvent.addInfo("DIAGNOSTICS", getDiagnostics());
+ tEvent.addInfo("FINISHED_MAPS", getFinishedMaps());
+ tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
index bc046c7..36737e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful completion of a map attempt
@@ -33,7 +36,7 @@ import org.apache.hadoop.mapreduce.TaskType;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class MapAttemptFinishedEvent implements HistoryEvent {
+public class MapAttemptFinishedEvent implements HistoryEvent {
private MapAttemptFinished datum = null;
@@ -218,4 +221,23 @@ public class MapAttemptFinishedEvent implements HistoryEvent {
return physMemKbytes;
}
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("STATE", getState());
+ tEvent.addInfo("MAP_FINISH_TIME", getMapFinishTime());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("HOSTNAME", getHostname());
+ tEvent.addInfo("PORT", getPort());
+ tEvent.addInfo("RACK_NAME", getRackName());
+ tEvent.addInfo("ATTEMPT_ID", getAttemptId() == null ?
+ "" : getAttemptId().toString());
+ return tEvent;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
index b8f049c..daa454c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record the normalized map/reduce requirements.
@@ -71,4 +73,13 @@ public class NormalizedResourceEvent implements HistoryEvent {
public void setDatum(Object datum) {
throw new UnsupportedOperationException("Not a seriable object");
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("MEMORY", "" + getMemory());
+ tEvent.addInfo("TASK_TYPE", getTaskType());
+ return tEvent;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
index 6644a48..6087c7a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
@@ -26,6 +26,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful completion of a reduce attempt
@@ -33,7 +36,7 @@ import org.apache.hadoop.mapreduce.TaskType;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class ReduceAttemptFinishedEvent implements HistoryEvent {
+public class ReduceAttemptFinishedEvent implements HistoryEvent {
private ReduceAttemptFinished datum = null;
@@ -222,5 +225,25 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent {
public int[] getPhysMemKbytes() {
return physMemKbytes;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("ATTEMPT_ID", getAttemptId() == null ?
+ "" : getAttemptId().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("STATE", getState());
+ tEvent.addInfo("SHUFFLE_FINISH_TIME", getShuffleFinishTime());
+ tEvent.addInfo("SORT_FINISH_TIME", getSortFinishTime());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("HOSTNAME", getHostname());
+ tEvent.addInfo("PORT", getPort());
+ tEvent.addInfo("RACK_NAME", getRackName());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
index bb7dbe0..c7c4387 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
@@ -25,6 +25,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record successful task completion
@@ -135,5 +138,21 @@ public class TaskAttemptFinishedEvent implements HistoryEvent {
? EventType.MAP_ATTEMPT_FINISHED
: EventType.REDUCE_ATTEMPT_FINISHED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("ATTEMPT_ID", getAttemptId() == null ?
+ "" : getAttemptId().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("STATE", getState());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("HOSTNAME", getHostname());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
index c8c250a..d8baec4 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
@@ -23,8 +23,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -132,5 +134,21 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
}
return null;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("TASK_ATTEMPT_ID",
+ getTaskAttemptId().toString());
+ tEvent.addInfo("START_TIME", getStartTime());
+ tEvent.addInfo("HTTP_PORT", getHttpPort());
+ tEvent.addInfo("TRACKER_NAME", getTrackerName());
+ tEvent.addInfo("SHUFFLE_PORT", getShufflePort());
+ tEvent.addInfo("CONTAINER_ID", getContainerId() == null ?
+ "" : getContainerId().toString());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
index 77ee2a0..0bb1358 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
@@ -25,6 +25,9 @@ import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.mapred.ProgressSplitsBlock;
@@ -247,5 +250,26 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
public int[] getPhysMemKbytes() {
return physMemKbytes;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("TASK_ATTEMPT_ID", getTaskAttemptId() == null ?
+ "" : getTaskAttemptId().toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("ERROR", getError());
+ tEvent.addInfo("STATUS", getTaskStatus());
+ tEvent.addInfo("HOSTNAME", getHostname());
+ tEvent.addInfo("PORT", getPort());
+ tEvent.addInfo("RACK_NAME", getRackName());
+ tEvent.addInfo("SHUFFLE_FINISH_TIME", getFinishTime());
+ tEvent.addInfo("SORT_FINISH_TIME", getFinishTime());
+ tEvent.addInfo("MAP_FINISH_TIME", getFinishTime());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
index 2838f08..5e82dea 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
@@ -20,10 +20,14 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -136,5 +140,20 @@ public class TaskFailedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.TASK_FAILED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("STATUS", TaskStatus.State.FAILED.toString());
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("ERROR", getError());
+ tEvent.addInfo("FAILED_ATTEMPT_ID",
+ getFailedAttemptID() == null ? "" : getFailedAttemptID().toString());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
index d4ec74d..e359e32 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
@@ -21,10 +21,14 @@ package org.apache.hadoop.mapreduce.jobhistory;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record the successful completion of a task
@@ -115,5 +119,19 @@ public class TaskFinishedEvent implements HistoryEvent {
return EventType.TASK_FINISHED;
}
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("COUNTERS_GROUPS",
+ JobHistoryEventUtils.countersToJSON(getCounters()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ tEvent.addInfo("STATUS", TaskStatus.State.SUCCEEDED.toString());
+ tEvent.addInfo("SUCCESSFUL_TASK_ATTEMPT_ID",
+ getSuccessfulTaskAttemptId() == null ? "" :
+ getSuccessfulTaskAttemptId().toString());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
index ed53b03..d1b97bf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
@@ -23,6 +23,8 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
/**
* Event to record the start of a task
@@ -71,5 +73,15 @@ public class TaskStartedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.TASK_STARTED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("TASK_TYPE", getTaskType().toString());
+ tEvent.addInfo("START_TIME", getStartTime());
+ tEvent.addInfo("SPLIT_LOCATIONS", getSplitLocations());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
index 58f4143..b9a389c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
@@ -23,6 +23,8 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.avro.util.Utf8;
@@ -60,5 +62,13 @@ public class TaskUpdatedEvent implements HistoryEvent {
public EventType getEventType() {
return EventType.TASK_UPDATED;
}
+
+ @Override
+ public TimelineEvent toTimelineEvent() {
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
+ tEvent.addInfo("FINISH_TIME", getFinishTime());
+ return tEvent;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
new file mode 100644
index 0000000..f4896ff
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -0,0 +1,51 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.mapreduce.util;
+
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+
+import org.codehaus.jackson.JsonNode;
+import org.codehaus.jackson.map.ObjectMapper;
+import org.codehaus.jackson.node.ArrayNode;
+import org.codehaus.jackson.node.ObjectNode;
+
+public class JobHistoryEventUtils {
+
+ public static JsonNode countersToJSON(Counters counters) {
+ ObjectMapper mapper = new ObjectMapper();
+ ArrayNode nodes = mapper.createArrayNode();
+ if (counters != null) {
+ for (CounterGroup counterGroup : counters) {
+ ObjectNode groupNode = nodes.addObject();
+ groupNode.put("NAME", counterGroup.getName());
+ groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
+ ArrayNode countersNode = groupNode.putArray("COUNTERS");
+ for (Counter counter : counterGroup) {
+ ObjectNode counterNode = countersNode.addObject();
+ counterNode.put("NAME", counter.getName());
+ counterNode.put("DISPLAY_NAME", counter.getDisplayName());
+ counterNode.put("VALUE", counter.getValue());
+ }
+ }
+ }
+ return nodes;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index 6d205c5..c7f61cf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -607,6 +607,13 @@
</description>
</property>
+ <property>
+ <name>mapreduce.job.new-timeline-service.enabled</name>
+ <value>false</value>
+ <description>Specifies if posting job and task events to new timeline service.
+ </description>
+</property>
+
<property>
<name>mapreduce.input.fileinputformat.split.minsize</name>
<value>0</value>
[08/50] [abbrv] hadoop git commit: YARN-3333. Rename
TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee
Posted by sj...@apache.org.
YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee
(cherry picked from commit dda84085cabd8fdf143b380e54e1730802fd9912)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63c7210c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63c7210c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63c7210c
Branch: refs/heads/YARN-2928
Commit: 63c7210c248be9a8e65b249b0593f1ace6003db5
Parents: 32acd9b
Author: Junping Du <ju...@apache.org>
Authored: Thu Mar 19 11:49:07 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:38:43 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 5 +-
.../hadoop-yarn/hadoop-yarn-api/pom.xml | 4 +
.../api/protocolrecords/AllocateResponse.java | 20 +-
.../timelineservice/TimelineWriteResponse.java | 20 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 20 +-
.../src/main/proto/yarn_service_protos.proto | 2 +-
.../pom.xml | 10 +
.../distributedshell/ApplicationMaster.java | 54 ++--
.../applications/distributedshell/Client.java | 8 +-
.../distributedshell/TestDistributedShell.java | 10 +-
.../hadoop/yarn/client/api/AMRMClient.java | 6 +-
.../yarn/client/api/async/AMRMClientAsync.java | 4 +-
.../api/async/impl/AMRMClientAsyncImpl.java | 20 +-
.../impl/pb/AllocateResponsePBImpl.java | 16 +-
.../hadoop/yarn/client/api/TimelineClient.java | 2 +-
.../client/api/impl/TimelineClientImpl.java | 32 +--
.../src/main/resources/yarn-default.xml | 14 +-
.../hadoop/yarn/TestContainerLaunchRPC.java | 2 +-
.../hadoop/yarn/api/TestAllocateResponse.java | 12 +-
.../hadoop-yarn-server-common/pom.xml | 2 +-
.../api/AggregatorNodemanagerProtocol.java | 56 ----
.../api/AggregatorNodemanagerProtocolPB.java | 33 ---
.../api/CollectorNodemanagerProtocol.java | 57 ++++
.../api/CollectorNodemanagerProtocolPB.java | 33 +++
...gregatorNodemanagerProtocolPBClientImpl.java | 94 -------
...ollectorNodemanagerProtocolPBClientImpl.java | 94 +++++++
...regatorNodemanagerProtocolPBServiceImpl.java | 61 ----
...llectorNodemanagerProtocolPBServiceImpl.java | 59 ++++
.../protocolrecords/NodeHeartbeatRequest.java | 13 +-
.../protocolrecords/NodeHeartbeatResponse.java | 8 +-
.../ReportNewAggregatorsInfoRequest.java | 53 ----
.../ReportNewAggregatorsInfoResponse.java | 32 ---
.../ReportNewCollectorInfoRequest.java | 53 ++++
.../ReportNewCollectorInfoResponse.java | 32 +++
.../impl/pb/NodeHeartbeatRequestPBImpl.java | 58 ++--
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 60 ++--
.../ReportNewAggregatorsInfoRequestPBImpl.java | 142 ----------
.../ReportNewAggregatorsInfoResponsePBImpl.java | 74 -----
.../pb/ReportNewCollectorInfoRequestPBImpl.java | 142 ++++++++++
.../ReportNewCollectorInfoResponsePBImpl.java | 74 +++++
.../server/api/records/AppAggregatorsMap.java | 33 ---
.../server/api/records/AppCollectorsMap.java | 46 +++
.../impl/pb/AppAggregatorsMapPBImpl.java | 151 ----------
.../records/impl/pb/AppCollectorsMapPBImpl.java | 151 ++++++++++
.../proto/aggregatornodemanager_protocol.proto | 29 --
.../proto/collectornodemanager_protocol.proto | 29 ++
.../yarn_server_common_service_protos.proto | 18 +-
.../java/org/apache/hadoop/yarn/TestRPC.java | 116 ++++----
.../hadoop/yarn/TestYarnServerApiClasses.java | 24 +-
.../hadoop/yarn/server/nodemanager/Context.java | 14 +-
.../yarn/server/nodemanager/NodeManager.java | 56 ++--
.../nodemanager/NodeStatusUpdaterImpl.java | 11 +-
.../aggregatormanager/NMAggregatorService.java | 113 --------
.../collectormanager/NMCollectorService.java | 110 ++++++++
.../application/ApplicationImpl.java | 9 +-
.../ApplicationMasterService.java | 12 +-
.../resourcemanager/ResourceTrackerService.java | 72 ++---
.../server/resourcemanager/rmapp/RMApp.java | 22 +-
.../rmapp/RMAppAggregatorUpdateEvent.java | 36 ---
.../rmapp/RMAppCollectorUpdateEvent.java | 37 +++
.../resourcemanager/rmapp/RMAppEventType.java | 4 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 60 ++--
.../applicationsmanager/MockAsm.java | 6 +-
.../server/resourcemanager/rmapp/MockRMApp.java | 8 +-
.../hadoop-yarn-server-tests/pom.xml | 5 +
.../TestTimelineServiceClientIntegration.java | 52 +++-
.../hadoop-yarn-server-timelineservice/pom.xml | 10 +
.../aggregator/AppLevelTimelineAggregator.java | 57 ----
.../PerNodeTimelineAggregatorsAuxService.java | 211 --------------
.../aggregator/TimelineAggregator.java | 122 --------
.../TimelineAggregatorWebService.java | 180 ------------
.../TimelineAggregatorsCollection.java | 271 ------------------
.../collector/AppLevelTimelineCollector.java | 57 ++++
.../PerNodeTimelineCollectorsAuxService.java | 214 ++++++++++++++
.../collector/TimelineCollector.java | 122 ++++++++
.../collector/TimelineCollectorManager.java | 278 +++++++++++++++++++
.../collector/TimelineCollectorWebService.java | 183 ++++++++++++
.../storage/FileSystemTimelineWriterImpl.java | 6 +-
.../TestAppLevelTimelineAggregator.java | 23 --
...estPerNodeTimelineAggregatorsAuxService.java | 150 ----------
.../TestTimelineAggregatorsCollection.java | 109 --------
.../TestAppLevelTimelineCollector.java | 23 ++
...TestPerNodeTimelineCollectorsAuxService.java | 164 +++++++++++
.../collector/TestTimelineCollectorManager.java | 118 ++++++++
.../TestFileSystemTimelineWriterImpl.java | 43 ++-
85 files changed, 2588 insertions(+), 2468 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index aa73ce3..42da7bf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -32,13 +32,16 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3039. Implemented the app-level timeline aggregator discovery service.
(Junping Du via zjshen)
+ YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. (Sangjin Lee
+ via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
BUG FIXES
-Trunk - Unreleased
+Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index 5c4156b..ed74a44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -45,6 +45,10 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</dependency>
+ <dependency>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index 421c2a0..6703249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -120,7 +120,7 @@ public abstract class AllocateResponse {
response.setAMRMToken(amRMToken);
return response;
}
-
+
@Public
@Unstable
public static AllocateResponse newInstance(int responseId,
@@ -130,13 +130,13 @@ public abstract class AllocateResponse {
PreemptionMessage preempt, List<NMToken> nmTokens, Token amRMToken,
List<ContainerResourceIncrease> increasedContainers,
List<ContainerResourceDecrease> decreasedContainers,
- String aggregatorAddr) {
+ String collectorAddr) {
AllocateResponse response =
newInstance(responseId, completedContainers, allocatedContainers,
updatedNodes, availResources, command, numClusterNodes, preempt,
nmTokens, increasedContainers, decreasedContainers);
response.setAMRMToken(amRMToken);
- response.setAggregatorAddr(aggregatorAddr);
+ response.setCollectorAddr(collectorAddr);
return response;
}
@@ -323,18 +323,18 @@ public abstract class AllocateResponse {
@Private
@Unstable
public abstract void setAMRMToken(Token amRMToken);
-
+
/**
- * The address of aggregator that belong to this app
+ * The address of collector that belong to this app
*
- * @return The address of aggregator that belong to this attempt
+ * @return The address of collector that belong to this attempt
*/
@Public
@Unstable
- public abstract String getAggregatorAddr();
-
+ public abstract String getCollectorAddr();
+
@Private
@Unstable
- public abstract void setAggregatorAddr(String aggregatorAddr);
-
+ public abstract void setCollectorAddr(String collectorAddr);
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
index 82ecdbd..4739d8f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
@@ -47,7 +47,7 @@ public class TimelineWriteResponse {
/**
* Get a list of {@link TimelineWriteError} instances
- *
+ *
* @return a list of {@link TimelineWriteError} instances
*/
@XmlElement(name = "errors")
@@ -57,7 +57,7 @@ public class TimelineWriteResponse {
/**
* Add a single {@link TimelineWriteError} instance into the existing list
- *
+ *
* @param error
* a single {@link TimelineWriteError} instance
*/
@@ -67,7 +67,7 @@ public class TimelineWriteResponse {
/**
* Add a list of {@link TimelineWriteError} instances into the existing list
- *
+ *
* @param errors
* a list of {@link TimelineWriteError} instances
*/
@@ -77,7 +77,7 @@ public class TimelineWriteResponse {
/**
* Set the list to the given list of {@link TimelineWriteError} instances
- *
+ *
* @param errors
* a list of {@link TimelineWriteError} instances
*/
@@ -107,7 +107,7 @@ public class TimelineWriteResponse {
/**
* Get the entity Id
- *
+ *
* @return the entity Id
*/
@XmlElement(name = "entity")
@@ -117,7 +117,7 @@ public class TimelineWriteResponse {
/**
* Set the entity Id
- *
+ *
* @param entityId
* the entity Id
*/
@@ -127,7 +127,7 @@ public class TimelineWriteResponse {
/**
* Get the entity type
- *
+ *
* @return the entity type
*/
@XmlElement(name = "entitytype")
@@ -137,7 +137,7 @@ public class TimelineWriteResponse {
/**
* Set the entity type
- *
+ *
* @param entityType
* the entity type
*/
@@ -147,7 +147,7 @@ public class TimelineWriteResponse {
/**
* Get the error code
- *
+ *
* @return an error code
*/
@XmlElement(name = "errorcode")
@@ -157,7 +157,7 @@ public class TimelineWriteResponse {
/**
* Set the error code to the given error code
- *
+ *
* @param errorCode
* an error code
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6874eec..9d5b63b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -672,10 +672,10 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT = 20;
/** Number of threads container manager uses.*/
- public static final String NM_AGGREGATOR_SERVICE_THREAD_COUNT =
- NM_PREFIX + "aggregator-service.thread-count";
- public static final int DEFAULT_NM_AGGREGATOR_SERVICE_THREAD_COUNT = 5;
-
+ public static final String NM_COLLECTOR_SERVICE_THREAD_COUNT =
+ NM_PREFIX + "collector-service.thread-count";
+ public static final int DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT = 5;
+
/** Number of threads used in cleanup.*/
public static final String NM_DELETE_THREAD_COUNT =
NM_PREFIX + "delete.thread-count";
@@ -703,13 +703,13 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_NM_LOCALIZER_ADDRESS = "0.0.0.0:" +
DEFAULT_NM_LOCALIZER_PORT;
- /** Address where the aggregator service IPC is.*/
- public static final String NM_AGGREGATOR_SERVICE_ADDRESS =
- NM_PREFIX + "aggregator-service.address";
- public static final int DEFAULT_NM_AGGREGATOR_SERVICE_PORT = 8048;
- public static final String DEFAULT_NM_AGGREGATOR_SERVICE_ADDRESS =
+ /** Address where the collector service IPC is.*/
+ public static final String NM_COLLECTOR_SERVICE_ADDRESS =
+ NM_PREFIX + "collector-service.address";
+ public static final int DEFAULT_NM_COLLECTOR_SERVICE_PORT = 8048;
+ public static final String DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS =
"0.0.0.0:" + DEFAULT_NM_LOCALIZER_PORT;
-
+
/** Interval in between cache cleanups.*/
public static final String NM_LOCALIZER_CACHE_CLEANUP_INTERVAL_MS =
NM_PREFIX + "localizer.cache.cleanup.interval-ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 40b8bfa..23d6e75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -87,7 +87,7 @@ message AllocateResponseProto {
repeated ContainerResourceIncreaseProto increased_containers = 10;
repeated ContainerResourceDecreaseProto decreased_containers = 11;
optional hadoop.common.TokenProto am_rm_token = 12;
- optional string aggregator_addr = 13;
+ optional string collector_addr = 13;
}
enum SchedulerResourceTypes {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 09a56ea..713f12b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -70,6 +70,16 @@
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index b728502..e107731 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -39,10 +39,10 @@ import java.util.Set;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.GnuParser;
@@ -216,12 +216,12 @@ public class ApplicationMaster {
private int appMasterRpcPort = -1;
// Tracking url to which app master publishes info for clients to monitor
private String appMasterTrackingUrl = "";
-
+
private boolean newTimelineService = false;
-
+
// For posting entities in new timeline service in a non-blocking way
// TODO replace with event loop in TimelineClient.
- private static ExecutorService threadPool =
+ private static ExecutorService threadPool =
Executors.newCachedThreadPool(
new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
.build());
@@ -314,9 +314,9 @@ public class ApplicationMaster {
}
appMaster.run();
result = appMaster.finish();
-
+
threadPool.shutdown();
-
+
while (!threadPool.isTerminated()) { // wait for all posting thread to finish
try {
if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) {
@@ -398,7 +398,7 @@ public class ApplicationMaster {
"No. of containers on which the shell command needs to be executed");
opts.addOption("priority", true, "Application Priority. Default 0");
opts.addOption("debug", false, "Dump out debug information");
- opts.addOption("timeline_service_version", true,
+ opts.addOption("timeline_service_version", true,
"Version for timeline service");
opts.addOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().parse(opts, args);
@@ -539,7 +539,7 @@ public class ApplicationMaster {
if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
if (cliParser.hasOption("timeline_service_version")) {
- String timelineServiceVersion =
+ String timelineServiceVersion =
cliParser.getOptionValue("timeline_service_version", "v1");
if (timelineServiceVersion.trim().equalsIgnoreCase("v1")) {
newTimelineService = false;
@@ -605,12 +605,12 @@ public class ApplicationMaster {
appSubmitterUgi =
UserGroupInformation.createRemoteUser(appSubmitterUserName);
appSubmitterUgi.addCredentials(credentials);
-
+
AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler();
amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener);
amRMClient.init(conf);
amRMClient.start();
-
+
containerListener = createNMCallbackHandler();
nmClientAsync = new NMClientAsyncImpl(containerListener);
nmClientAsync.init(conf);
@@ -795,7 +795,7 @@ public class ApplicationMaster {
if(timelineClient != null) {
timelineClient.stop();
}
-
+
return success;
}
@@ -1291,11 +1291,11 @@ public class ApplicationMaster {
}
private static void publishContainerStartEventOnNewTimelineService(
- final TimelineClient timelineClient, final Container container,
+ final TimelineClient timelineClient, final Container container,
final String domainId, final UserGroupInformation ugi) {
Runnable publishWrapper = new Runnable() {
public void run() {
- publishContainerStartEventOnNewTimelineServiceBase(timelineClient,
+ publishContainerStartEventOnNewTimelineServiceBase(timelineClient,
container, domainId, ugi);
}
};
@@ -1305,14 +1305,14 @@ public class ApplicationMaster {
private static void publishContainerStartEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, Container container, String domainId,
UserGroupInformation ugi) {
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName());
-
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setId(DSEvent.DS_CONTAINER_START.toString());
@@ -1334,29 +1334,29 @@ public class ApplicationMaster {
e instanceof UndeclaredThrowableException ? e.getCause() : e);
}
}
-
+
private static void publishContainerEndEventOnNewTimelineService(
final TimelineClient timelineClient, final ContainerStatus container,
final String domainId, final UserGroupInformation ugi) {
Runnable publishWrapper = new Runnable() {
public void run() {
- publishContainerEndEventOnNewTimelineServiceBase(timelineClient,
+ publishContainerEndEventOnNewTimelineServiceBase(timelineClient,
container, domainId, ugi);
}
};
threadPool.execute(publishWrapper);
}
-
+
private static void publishContainerEndEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, final ContainerStatus container,
final String domainId, final UserGroupInformation ugi) {
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getContainerId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName());
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
event.setId(DSEvent.DS_CONTAINER_END.toString());
@@ -1381,28 +1381,28 @@ public class ApplicationMaster {
private static void publishApplicationAttemptEventOnNewTimelineService(
final TimelineClient timelineClient, final String appAttemptId,
- final DSEvent appEvent, final String domainId,
+ final DSEvent appEvent, final String domainId,
final UserGroupInformation ugi) {
-
+
Runnable publishWrapper = new Runnable() {
public void run() {
- publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient,
+ publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient,
appAttemptId, appEvent, domainId, ugi);
}
};
threadPool.execute(publishWrapper);
}
-
+
private static void publishApplicationAttemptEventOnNewTimelineServiceBase(
final TimelineClient timelineClient, String appAttemptId,
DSEvent appEvent, String domainId, UserGroupInformation ugi) {
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(appAttemptId);
entity.setType(DSEntity.DS_APP_ATTEMPT.toString());
//entity.setDomainId(domainId);
entity.addInfo("user", ugi.getShortUserName());
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setId(appEvent.toString());
event.setTimestamp(System.currentTimeMillis());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 9f7f9c0..163f8f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -185,7 +185,7 @@ public class Client {
// Command line options
private Options opts;
-
+
private String timelineServiceVersion;
private static final String shellCommandPath = "shellCommands";
@@ -357,11 +357,11 @@ public class Client {
throw new IllegalArgumentException("Invalid virtual cores specified for application master, exiting."
+ " Specified virtual cores=" + amVCores);
}
-
+
if (cliParser.hasOption("timeline_service_version")) {
- timelineServiceVersion =
+ timelineServiceVersion =
cliParser.getOptionValue("timeline_service_version", "v1");
- if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") ||
+ if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") ||
timelineServiceVersion.trim().equalsIgnoreCase("v2"))) {
throw new IllegalArgumentException(
"timeline_service_version is not set properly, should be 'v1' or 'v2'");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 97d9168..0af050c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -50,7 +50,7 @@ import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
-import org.apache.hadoop.yarn.server.timelineservice.aggregator.PerNodeTimelineAggregatorsAuxService;
+import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.junit.After;
import org.junit.Assert;
@@ -67,7 +67,7 @@ public class TestDistributedShell {
protected MiniYARNCluster yarnCluster = null;
protected YarnConfiguration conf = null;
private static final int NUM_NMS = 1;
- private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_aggregator";
+ private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
protected final static String APPMASTER_JAR =
JarFinder.getJar(ApplicationMaster.class);
@@ -91,14 +91,14 @@ public class TestDistributedShell {
// mark if we need to launch the v1 timeline server
boolean enableATSV1 = false;
if (!currTestName.getMethodName().toLowerCase().contains("v2")) {
- // disable aux-service based timeline aggregators
+ // disable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
enableATSV1 = true;
} else {
- // enable aux-service based timeline aggregators
+ // enable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
- + ".class", PerNodeTimelineAggregatorsAuxService.class.getName());
+ + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
}
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 56f2b10..afb2e09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -47,7 +47,7 @@ import com.google.common.collect.ImmutableList;
public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
AbstractService {
private static final Log LOG = LogFactory.getLog(AMRMClient.class);
-
+
private TimelineClient timelineClient;
/**
@@ -382,7 +382,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
public void registerTimelineClient(TimelineClient timelineClient) {
this.timelineClient = timelineClient;
}
-
+
/**
* Get registered timeline client.
* @return
@@ -390,7 +390,7 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
public TimelineClient getRegisteredTimeineClient() {
return this.timelineClient;
}
-
+
/**
* Wait for <code>check</code> to return true for each 1000 ms.
* See also {@link #waitFor(com.google.common.base.Supplier, int)}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
index be5610e..1a5c257 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/AMRMClientAsync.java
@@ -194,7 +194,7 @@ extends AbstractService {
* @return Current number of nodes in the cluster
*/
public abstract int getClusterNodeCount();
-
+
/**
* Register TimelineClient to AMRMClient.
* @param timelineClient
@@ -202,7 +202,7 @@ extends AbstractService {
public void registerTimelineClient(TimelineClient timelineClient) {
client.registerTimelineClient(timelineClient);
}
-
+
/**
* Get registered timeline client.
* @return
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index f0f0bc9..5351ef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -66,8 +66,8 @@ extends AMRMClientAsync<T> {
private volatile boolean keepRunning;
private volatile float progress;
- private volatile String aggregatorAddr;
-
+ private volatile String collectorAddr;
+
private volatile Throwable savedException;
public AMRMClientAsyncImpl(int intervalMs, CallbackHandler callbackHandler) {
@@ -307,15 +307,15 @@ extends AMRMClientAsync<T> {
if (!allocated.isEmpty()) {
handler.onContainersAllocated(allocated);
}
-
- String aggregatorAddress = response.getAggregatorAddr();
+
+ String collectorAddress = response.getCollectorAddr();
TimelineClient timelineClient = client.getRegisteredTimeineClient();
- if (timelineClient != null && aggregatorAddress != null
- && !aggregatorAddress.isEmpty()) {
- if (aggregatorAddr == null ||
- !aggregatorAddr.equals(aggregatorAddress)) {
- aggregatorAddr = aggregatorAddress;
- timelineClient.setTimelineServiceAddress(aggregatorAddress);
+ if (timelineClient != null && collectorAddress != null
+ && !collectorAddress.isEmpty()) {
+ if (collectorAddr == null ||
+ !collectorAddr.equals(collectorAddress)) {
+ collectorAddr = collectorAddress;
+ timelineClient.setTimelineServiceAddress(collectorAddress);
}
}
progress = handler.getProgress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index 605c29a..8dc473c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -384,22 +384,22 @@ public class AllocateResponsePBImpl extends AllocateResponse {
}
this.amrmToken = amRMToken;
}
-
+
@Override
- public String getAggregatorAddr() {
+ public String getCollectorAddr() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
- return p.getAggregatorAddr();
+ return p.getCollectorAddr();
}
-
+
@Override
- public void setAggregatorAddr(String aggregatorAddr) {
+ public void setCollectorAddr(String collectorAddr) {
maybeInitBuilder();
- if (aggregatorAddr == null) {
- builder.clearAggregatorAddr();
+ if (collectorAddr == null) {
+ builder.clearCollectorAddr();
return;
}
- builder.setAggregatorAddr(aggregatorAddr);
+ builder.setCollectorAddr(collectorAddr);
}
private synchronized void initLocalIncreasedContainerList() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index 27ef3e4..f2707ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -195,5 +195,5 @@ public abstract class TimelineClient extends AbstractService {
* the timeline service address
*/
public abstract void setTimelineServiceAddress(String address);
-
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 8992422..c722edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -116,14 +116,14 @@ public class TimelineClientImpl extends TimelineClient {
private URI resURI;
private UserGroupInformation authUgi;
private String doAsUser;
-
+
private volatile String timelineServiceAddress;
-
+
// Retry parameters for identifying new timeline service
// TODO consider to merge with connection retry
private int maxServiceRetries;
private long serviceRetryInterval;
-
+
private boolean newTimelineService = false;
@Private
@@ -323,7 +323,7 @@ public class TimelineClientImpl extends TimelineClient {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS));
}
LOG.info("Timeline service address: " + getTimelineServiceAddress());
- }
+ }
super.serviceInit(conf);
}
@@ -374,16 +374,16 @@ public class TimelineClientImpl extends TimelineClient {
YarnException {
doPosting(domain, "domain");
}
-
+
// Used for new timeline service only
@Private
- public void putObjects(String path, MultivaluedMap<String, String> params,
+ public void putObjects(String path, MultivaluedMap<String, String> params,
Object obj) throws IOException, YarnException {
-
- // timelineServiceAddress could haven't be initialized yet
+
+ // timelineServiceAddress could haven't be initialized yet
// or stale (only for new timeline service)
int retries = pollTimelineServiceAddress(this.maxServiceRetries);
-
+
// timelineServiceAddress could be stale, add retry logic here.
boolean needRetry = true;
while (needRetry) {
@@ -400,13 +400,13 @@ public class TimelineClientImpl extends TimelineClient {
}
}
}
-
+
/**
* Check if reaching to maximum of retries.
* @param retries
* @param e
*/
- private void checkRetryWithSleep(int retries, Exception e) throws
+ private void checkRetryWithSleep(int retries, Exception e) throws
YarnException, IOException {
if (retries > 0) {
try {
@@ -416,8 +416,8 @@ public class TimelineClientImpl extends TimelineClient {
}
} else {
LOG.error(
- "TimelineClient has reached to max retry times :" +
- this.maxServiceRetries + " for service address: " +
+ "TimelineClient has reached to max retry times :" +
+ this.maxServiceRetries + " for service address: " +
timelineServiceAddress);
if (e instanceof YarnException) {
throw (YarnException)e;
@@ -488,12 +488,12 @@ public class TimelineClientImpl extends TimelineClient {
}
return resp;
}
-
+
@Override
public void setTimelineServiceAddress(String address) {
this.timelineServiceAddress = address;
}
-
+
private String getTimelineServiceAddress() {
return this.timelineServiceAddress;
}
@@ -642,7 +642,7 @@ public class TimelineClientImpl extends TimelineClient {
throw new YarnRuntimeException("Unknown resource type");
}
}
-
+
/**
* Poll TimelineServiceAddress for maximum of retries times if it is null
* @param retries
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 85525a2..78b6ae8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -927,10 +927,10 @@
<name>yarn.nodemanager.container-manager.thread-count</name>
<value>20</value>
</property>
-
+
<property>
- <description>Number of threads aggregator service uses.</description>
- <name>yarn.nodemanager.aggregator-service.thread-count</name>
+ <description>Number of threads collector service uses.</description>
+ <name>yarn.nodemanager.collector-service.thread-count</name>
<value>5</value>
</property>
@@ -1001,11 +1001,11 @@
<name>yarn.nodemanager.localizer.address</name>
<value>${yarn.nodemanager.hostname}:8040</value>
</property>
-
-
+
+
<property>
- <description>Address where the aggregator service IPC is.</description>
- <name>yarn.nodemanager.aggregator-service.address</name>
+ <description>Address where the collector service IPC is.</description>
+ <name>yarn.nodemanager.collector-service.address</name>
<value>${yarn.nodemanager.hostname}:8048</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
index 26d6d04..515adc8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/TestContainerLaunchRPC.java
@@ -131,7 +131,7 @@ public class TestContainerLaunchRPC {
Assert.fail("timeout exception should have occurred!");
}
-
+
public static Token newContainerToken(NodeId nodeId, byte[] password,
ContainerTokenIdentifier tokenIdentifier) {
// RPC layer client expects ip:port as service for tokens
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
index ef0bdcc..5af0f11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestAllocateResponse.java
@@ -111,21 +111,21 @@ public class TestAllocateResponse {
Assert.assertEquals(0, r.getIncreasedContainers().size());
Assert.assertEquals(0, r.getDecreasedContainers().size());
}
-
+
@SuppressWarnings("deprecation")
@Test
- public void testAllocateResponseWithAggregatorAddress() {
- final String aggregatorAddr = "localhost:0";
+ public void testAllocateResponseWithCollectorAddress() {
+ final String collectorAddr = "localhost:0";
AllocateResponse r =
AllocateResponse.newInstance(3, new ArrayList<ContainerStatus>(),
new ArrayList<Container>(), new ArrayList<NodeReport>(), null,
- AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(), null,
- null, null, aggregatorAddr);
+ AMCommand.AM_RESYNC, 3, null, new ArrayList<NMToken>(), null,
+ null, null, collectorAddr);
AllocateResponseProto p = ((AllocateResponsePBImpl) r).getProto();
r = new AllocateResponsePBImpl(p);
// check value
- Assert.assertEquals(aggregatorAddr, r.getAggregatorAddr());
+ Assert.assertEquals(collectorAddr, r.getCollectorAddr());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 7148837..21ecd97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -142,7 +142,7 @@
<include>yarn_server_common_service_protos.proto</include>
<include>ResourceTracker.proto</include>
<include>SCMUploader.proto</include>
- <include>aggregatornodemanager_protocol.proto</include>
+ <include>collectornodemanager_protocol.proto</include>
</includes>
</source>
<output>${project.build.directory}/generated-sources/java</output>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java
deleted file mode 100644
index 53bdb4e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocol.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse;
-
-/**
- * <p>The protocol between an <code>TimelineAggregatorsCollection</code> and a
- * <code>NodeManager</code> to report a new application aggregator get launched.
- * </p>
- *
- */
-@Private
-public interface AggregatorNodemanagerProtocol {
-
- /**
- *
- * <p>
- * The <code>TimelineAggregatorsCollection</code> provides a list of mapping
- * between application and aggregator's address in
- * {@link ReportNewAggregatorsInfoRequest} to a <code>NodeManager</code> to
- * <em>register</em> aggregator's info, include: applicationId and REST URI to
- * access aggregator. NodeManager will add them into registered aggregators
- * and register them into <code>ResourceManager</code> afterwards.
- * </p>
- *
- * @param request the request of registering a new aggregator or a list of aggregators
- * @return
- * @throws YarnException
- * @throws IOException
- */
- ReportNewAggregatorsInfoResponse reportNewAggregatorInfo(
- ReportNewAggregatorsInfoRequest request)
- throws YarnException, IOException;
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java
deleted file mode 100644
index 4df80a5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/AggregatorNodemanagerProtocolPB.java
+++ /dev/null
@@ -1,33 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.ipc.ProtocolInfo;
-import org.apache.hadoop.yarn.proto.AggregatorNodemanagerProtocol.AggregatorNodemanagerProtocolService;
-
-@Private
-@Unstable
-@ProtocolInfo(
- protocolName = "org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB",
- protocolVersion = 1)
-public interface AggregatorNodemanagerProtocolPB extends
- AggregatorNodemanagerProtocolService.BlockingInterface {
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
new file mode 100644
index 0000000..26c121a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocol.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+
+/**
+ * <p>The protocol between an <code>TimelineCollectorManager</code> and a
+ * <code>NodeManager</code> to report a new application collector get launched.
+ * </p>
+ *
+ */
+@Private
+public interface CollectorNodemanagerProtocol {
+
+ /**
+ *
+ * <p>
+ * The <code>TimelineCollectorManager</code> provides a list of mapping
+ * between application and collector's address in
+ * {@link ReportNewCollectorInfoRequest} to a <code>NodeManager</code> to
+ * <em>register</em> collector's info, include: applicationId and REST URI to
+ * access collector. NodeManager will add them into registered collectors
+ * and register them into <code>ResourceManager</code> afterwards.
+ * </p>
+ *
+ * @param request the request of registering a new collector or a list of
+ * collectors
+ * @return
+ * @throws YarnException
+ * @throws IOException
+ */
+ ReportNewCollectorInfoResponse reportNewCollectorInfo(
+ ReportNewCollectorInfoRequest request)
+ throws YarnException, IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java
new file mode 100644
index 0000000..655e989
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/CollectorNodemanagerProtocolPB.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.yarn.proto.CollectorNodemanagerProtocol.CollectorNodemanagerProtocolService;
+
+@Private
+@Unstable
+@ProtocolInfo(
+ protocolName = "org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB",
+ protocolVersion = 1)
+public interface CollectorNodemanagerProtocolPB extends
+ CollectorNodemanagerProtocolService.BlockingInterface {
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java
deleted file mode 100644
index 6e777e7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/AggregatorNodemanagerProtocolPBClientImpl.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api.impl.pb.client;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.ipc.RPCUtil;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoResponsePBImpl;
-
-import com.google.protobuf.ServiceException;
-
-public class AggregatorNodemanagerProtocolPBClientImpl implements
- AggregatorNodemanagerProtocol, Closeable {
-
- // Not a documented config. Only used for tests internally
- static final String NM_COMMAND_TIMEOUT = YarnConfiguration.YARN_PREFIX
- + "rpc.nm-command-timeout";
-
- /**
- * Maximum of 1 minute timeout for a Node to react to the command
- */
- static final int DEFAULT_COMMAND_TIMEOUT = 60000;
-
- private AggregatorNodemanagerProtocolPB proxy;
-
- @Private
- public AggregatorNodemanagerProtocolPBClientImpl(long clientVersion,
- InetSocketAddress addr, Configuration conf) throws IOException {
- RPC.setProtocolEngine(conf, AggregatorNodemanagerProtocolPB.class,
- ProtobufRpcEngine.class);
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
-
- int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT);
- proxy =
- (AggregatorNodemanagerProtocolPB) RPC.getProxy(
- AggregatorNodemanagerProtocolPB.class,
- clientVersion, addr, ugi, conf,
- NetUtils.getDefaultSocketFactory(conf), expireIntvl);
- }
-
- @Override
- public ReportNewAggregatorsInfoResponse reportNewAggregatorInfo(
- ReportNewAggregatorsInfoRequest request) throws YarnException, IOException {
-
- ReportNewAggregatorsInfoRequestProto requestProto =
- ((ReportNewAggregatorsInfoRequestPBImpl) request).getProto();
- try {
- return new ReportNewAggregatorsInfoResponsePBImpl(
- proxy.reportNewAggregatorInfo(null, requestProto));
- } catch (ServiceException e) {
- RPCUtil.unwrapAndThrowException(e);
- return null;
- }
- }
-
- @Override
- public void close() {
- if (this.proxy != null) {
- RPC.stopProxy(this.proxy);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
new file mode 100644
index 0000000..276a540
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/client/CollectorNodemanagerProtocolPBClientImpl.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.impl.pb.client;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.RPCUtil;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl;
+
+import com.google.protobuf.ServiceException;
+
+public class CollectorNodemanagerProtocolPBClientImpl implements
+ CollectorNodemanagerProtocol, Closeable {
+
+ // Not a documented config. Only used for tests internally
+ static final String NM_COMMAND_TIMEOUT = YarnConfiguration.YARN_PREFIX
+ + "rpc.nm-command-timeout";
+
+ /**
+ * Maximum of 1 minute timeout for a Node to react to the command
+ */
+ static final int DEFAULT_COMMAND_TIMEOUT = 60000;
+
+ private CollectorNodemanagerProtocolPB proxy;
+
+ @Private
+ public CollectorNodemanagerProtocolPBClientImpl(long clientVersion,
+ InetSocketAddress addr, Configuration conf) throws IOException {
+ RPC.setProtocolEngine(conf, CollectorNodemanagerProtocolPB.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+
+ int expireIntvl = conf.getInt(NM_COMMAND_TIMEOUT, DEFAULT_COMMAND_TIMEOUT);
+ proxy =
+ (CollectorNodemanagerProtocolPB) RPC.getProxy(
+ CollectorNodemanagerProtocolPB.class,
+ clientVersion, addr, ugi, conf,
+ NetUtils.getDefaultSocketFactory(conf), expireIntvl);
+ }
+
+ @Override
+ public ReportNewCollectorInfoResponse reportNewCollectorInfo(
+ ReportNewCollectorInfoRequest request) throws YarnException, IOException {
+
+ ReportNewCollectorInfoRequestProto requestProto =
+ ((ReportNewCollectorInfoRequestPBImpl) request).getProto();
+ try {
+ return new ReportNewCollectorInfoResponsePBImpl(
+ proxy.reportNewCollectorInfo(null, requestProto));
+ } catch (ServiceException e) {
+ RPCUtil.unwrapAndThrowException(e);
+ return null;
+ }
+ }
+
+ @Override
+ public void close() {
+ if (this.proxy != null) {
+ RPC.stopProxy(this.proxy);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java
deleted file mode 100644
index 87bce16..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/AggregatorNodemanagerProtocolPBServiceImpl.java
+++ /dev/null
@@ -1,61 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api.impl.pb.service;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewAggregatorsInfoResponseProto;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.AggregatorNodemanagerProtocolPB;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewAggregatorsInfoResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoRequestPBImpl;
-import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewAggregatorsInfoResponsePBImpl;
-
-import com.google.protobuf.RpcController;
-import com.google.protobuf.ServiceException;
-
-public class AggregatorNodemanagerProtocolPBServiceImpl implements
- AggregatorNodemanagerProtocolPB {
-
- private AggregatorNodemanagerProtocol real;
-
- public AggregatorNodemanagerProtocolPBServiceImpl(AggregatorNodemanagerProtocol impl) {
- this.real = impl;
- }
-
- @Override
- public ReportNewAggregatorsInfoResponseProto reportNewAggregatorInfo(
- RpcController arg0, ReportNewAggregatorsInfoRequestProto proto)
- throws ServiceException {
- ReportNewAggregatorsInfoRequestPBImpl request =
- new ReportNewAggregatorsInfoRequestPBImpl(proto);
- try {
- ReportNewAggregatorsInfoResponse response = real.reportNewAggregatorInfo(request);
- return ((ReportNewAggregatorsInfoResponsePBImpl)response).getProto();
- } catch (YarnException e) {
- throw new ServiceException(e);
- } catch (IOException e) {
- throw new ServiceException(e);
- }
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
new file mode 100644
index 0000000..3f42732
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/impl/pb/service/CollectorNodemanagerProtocolPBServiceImpl.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.api.impl.pb.service;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.ReportNewCollectorInfoResponseProto;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocolPB;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReportNewCollectorInfoResponsePBImpl;
+
+import com.google.protobuf.RpcController;
+import com.google.protobuf.ServiceException;
+
+public class CollectorNodemanagerProtocolPBServiceImpl implements
+ CollectorNodemanagerProtocolPB {
+
+ private CollectorNodemanagerProtocol real;
+
+ public CollectorNodemanagerProtocolPBServiceImpl(CollectorNodemanagerProtocol impl) {
+ this.real = impl;
+ }
+
+ @Override
+ public ReportNewCollectorInfoResponseProto reportNewCollectorInfo(
+ RpcController arg0, ReportNewCollectorInfoRequestProto proto)
+ throws ServiceException {
+ ReportNewCollectorInfoRequestPBImpl request =
+ new ReportNewCollectorInfoRequestPBImpl(proto);
+ try {
+ ReportNewCollectorInfoResponse response = real.reportNewCollectorInfo(request);
+ return ((ReportNewCollectorInfoResponsePBImpl)response).getProto();
+ } catch (YarnException e) {
+ throw new ServiceException(e);
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
index 09cb9e3..c795e55 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatRequest.java
@@ -43,11 +43,11 @@ public abstract class NodeHeartbeatRequest {
nodeHeartbeatRequest.setNodeLabels(nodeLabels);
return nodeHeartbeatRequest;
}
-
+
public static NodeHeartbeatRequest newInstance(NodeStatus nodeStatus,
MasterKey lastKnownContainerTokenMasterKey,
MasterKey lastKnownNMTokenMasterKey, Set<NodeLabel> nodeLabels,
- Map<ApplicationId, String> registeredAggregators) {
+ Map<ApplicationId, String> registeredCollectors) {
NodeHeartbeatRequest nodeHeartbeatRequest =
Records.newRecord(NodeHeartbeatRequest.class);
nodeHeartbeatRequest.setNodeStatus(nodeStatus);
@@ -56,7 +56,7 @@ public abstract class NodeHeartbeatRequest {
nodeHeartbeatRequest
.setLastKnownNMTokenMasterKey(lastKnownNMTokenMasterKey);
nodeHeartbeatRequest.setNodeLabels(nodeLabels);
- nodeHeartbeatRequest.setRegisteredAggregators(registeredAggregators);
+ nodeHeartbeatRequest.setRegisteredCollectors(registeredCollectors);
return nodeHeartbeatRequest;
}
@@ -78,7 +78,8 @@ public abstract class NodeHeartbeatRequest {
public abstract void setLogAggregationReportsForApps(
List<LogAggregationReport> logAggregationReportsForApps);
- // This tells RM registered aggregators' address info on this node
- public abstract Map<ApplicationId, String> getRegisteredAggregators();
- public abstract void setRegisteredAggregators(Map<ApplicationId, String> appAggregatorsMap);
+ // This tells RM registered collectors' address info on this node
+ public abstract Map<ApplicationId, String> getRegisteredCollectors();
+ public abstract void setRegisteredCollectors(Map<ApplicationId,
+ String> appCollectorsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 7019ad7..8d28d19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -35,10 +35,10 @@ public interface NodeHeartbeatResponse {
List<ContainerId> getContainersToBeRemovedFromNM();
List<ApplicationId> getApplicationsToCleanup();
-
- // This tells NM the aggregators' address info of related Apps
- Map<ApplicationId, String> getAppAggregatorsMap();
- void setAppAggregatorsMap(Map<ApplicationId, String> appAggregatorsMap);
+
+ // This tells NM the collectors' address info of related apps
+ Map<ApplicationId, String> getAppCollectorsMap();
+ void setAppCollectorsMap(Map<ApplicationId, String> appCollectorsMap);
void setResponseId(int responseId);
void setNodeAction(NodeAction action);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java
deleted file mode 100644
index ae538a2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoRequest.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api.protocolrecords;
-
-import java.util.List;
-import java.util.Arrays;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.server.api.records.AppAggregatorsMap;
-import org.apache.hadoop.yarn.util.Records;
-
-@Private
-public abstract class ReportNewAggregatorsInfoRequest {
-
- public static ReportNewAggregatorsInfoRequest newInstance(
- List<AppAggregatorsMap> appAggregatorsList) {
- ReportNewAggregatorsInfoRequest request =
- Records.newRecord(ReportNewAggregatorsInfoRequest.class);
- request.setAppAggregatorsList(appAggregatorsList);
- return request;
- }
-
- public static ReportNewAggregatorsInfoRequest newInstance(
- ApplicationId id, String aggregatorAddr) {
- ReportNewAggregatorsInfoRequest request =
- Records.newRecord(ReportNewAggregatorsInfoRequest.class);
- request.setAppAggregatorsList(
- Arrays.asList(AppAggregatorsMap.newInstance(id, aggregatorAddr)));
- return request;
- }
-
- public abstract List<AppAggregatorsMap> getAppAggregatorsList();
-
- public abstract void setAppAggregatorsList(
- List<AppAggregatorsMap> appAggregatorsList);
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java
deleted file mode 100644
index 3b847d6..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/ReportNewAggregatorsInfoResponse.java
+++ /dev/null
@@ -1,32 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.api.protocolrecords;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.yarn.util.Records;
-
-public abstract class ReportNewAggregatorsInfoResponse {
-
- @Private
- public static ReportNewAggregatorsInfoResponse newInstance() {
- ReportNewAggregatorsInfoResponse response =
- Records.newRecord(ReportNewAggregatorsInfoResponse.class);
- return response;
- }
-
-}
[05/50] [abbrv] hadoop git commit: YARN-3333. Rename
TimelineAggregator etc. to TimelineCollector. Contributed by Sangjin Lee
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
new file mode 100644
index 0000000..59ecef1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -0,0 +1,214 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import java.nio.ByteBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
+import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
+import org.apache.hadoop.yarn.server.api.AuxiliaryService;
+import org.apache.hadoop.yarn.server.api.ContainerContext;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * The top-level server for the per-node timeline collector manager. Currently
+ * it is defined as an auxiliary service to accommodate running within another
+ * daemon (e.g. node manager).
+ */
+@Private
+@Unstable
+public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
+ private static final Log LOG =
+ LogFactory.getLog(PerNodeTimelineCollectorsAuxService.class);
+ private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+
+ private final TimelineCollectorManager collectorManager;
+
+ public PerNodeTimelineCollectorsAuxService() {
+ // use the same singleton
+ this(TimelineCollectorManager.getInstance());
+ }
+
+ @VisibleForTesting PerNodeTimelineCollectorsAuxService(
+ TimelineCollectorManager collectorsManager) {
+ super("timeline_collector");
+ this.collectorManager = collectorsManager;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ collectorManager.init(conf);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ collectorManager.start();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ collectorManager.stop();
+ super.serviceStop();
+ }
+
+ // these methods can be used as the basis for future service methods if the
+ // per-node collector runs separate from the node manager
+ /**
+ * Creates and adds an app level collector for the specified application id.
+ * The collector is also initialized and started. If the service already
+ * exists, no new service is created.
+ *
+ * @return whether it was added successfully
+ */
+ public boolean addApplication(ApplicationId appId) {
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId.toString());
+ return (collectorManager.putIfAbsent(appId, collector)
+ == collector);
+ }
+
+ /**
+ * Removes the app level collector for the specified application id. The
+ * collector is also stopped as a result. If the collector does not exist, no
+ * change is made.
+ *
+ * @return whether it was removed successfully
+ */
+ public boolean removeApplication(ApplicationId appId) {
+ String appIdString = appId.toString();
+ return collectorManager.remove(appIdString);
+ }
+
+ /**
+ * Creates and adds an app level collector for the specified application id.
+ * The collector is also initialized and started. If the collector already
+ * exists, no new collector is created.
+ */
+ @Override
+ public void initializeContainer(ContainerInitializationContext context) {
+ // intercept the event of the AM container being created and initialize the
+ // app level collector service
+ if (isApplicationMaster(context)) {
+ ApplicationId appId = context.getContainerId().
+ getApplicationAttemptId().getApplicationId();
+ addApplication(appId);
+ }
+ }
+
+ /**
+ * Removes the app level collector for the specified application id. The
+ * collector is also stopped as a result. If the collector does not exist, no
+ * change is made.
+ */
+ @Override
+ public void stopContainer(ContainerTerminationContext context) {
+ // intercept the event of the AM container being stopped and remove the app
+ // level collector service
+ if (isApplicationMaster(context)) {
+ ApplicationId appId = context.getContainerId().
+ getApplicationAttemptId().getApplicationId();
+ removeApplication(appId);
+ }
+ }
+
+ private boolean isApplicationMaster(ContainerContext context) {
+ // TODO this is based on a (shaky) assumption that the container id (the
+ // last field of the full container id) for an AM is always 1
+ // we want to make this much more reliable
+ ContainerId containerId = context.getContainerId();
+ return containerId.getContainerId() == 1L;
+ }
+
+ @VisibleForTesting
+ boolean hasApplication(String appId) {
+ return collectorManager.containsKey(appId);
+ }
+
+ @Override
+ public void initializeApplication(ApplicationInitializationContext context) {
+ }
+
+ @Override
+ public void stopApplication(ApplicationTerminationContext context) {
+ }
+
+ @Override
+ public ByteBuffer getMetaData() {
+ // TODO currently it is not used; we can return a more meaningful data when
+ // we connect it with an AM
+ return ByteBuffer.allocate(0);
+ }
+
+ @VisibleForTesting
+ public static PerNodeTimelineCollectorsAuxService
+ launchServer(String[] args, TimelineCollectorManager collectorManager) {
+ Thread
+ .setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(
+ PerNodeTimelineCollectorsAuxService.class, args, LOG);
+ PerNodeTimelineCollectorsAuxService auxService = null;
+ try {
+ auxService = collectorManager == null ?
+ new PerNodeTimelineCollectorsAuxService() :
+ new PerNodeTimelineCollectorsAuxService(collectorManager);
+ ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService),
+ SHUTDOWN_HOOK_PRIORITY);
+ YarnConfiguration conf = new YarnConfiguration();
+ auxService.init(conf);
+ auxService.start();
+ } catch (Throwable t) {
+ LOG.fatal("Error starting PerNodeTimelineCollectorServer", t);
+ ExitUtil.terminate(-1, "Error starting PerNodeTimelineCollectorServer");
+ }
+ return auxService;
+ }
+
+ private static class ShutdownHook implements Runnable {
+ private final PerNodeTimelineCollectorsAuxService auxService;
+
+ public ShutdownHook(PerNodeTimelineCollectorsAuxService auxService) {
+ this.auxService = auxService;
+ }
+
+ public void run() {
+ auxService.stop();
+ }
+ }
+
+ public static void main(String[] args) {
+ launchServer(args, null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
new file mode 100644
index 0000000..6e20e69
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -0,0 +1,122 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
+/**
+ * Service that handles writes to the timeline service and writes them to the
+ * backing storage.
+ *
+ * Classes that extend this can add their own lifecycle management or
+ * customization of request handling.
+ */
+@Private
+@Unstable
+public abstract class TimelineCollector extends CompositeService {
+ private static final Log LOG = LogFactory.getLog(TimelineCollector.class);
+
+ private TimelineWriter writer;
+
+ public TimelineCollector(String name) {
+ super(name);
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ writer = ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
+ FileSystemTimelineWriterImpl.class,
+ TimelineWriter.class), conf);
+ writer.init(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ writer.stop();
+ }
+
+ public TimelineWriter getWriter() {
+ return writer;
+ }
+
+ /**
+ * Handles entity writes. These writes are synchronous and are written to the
+ * backing storage without buffering/batching. If any entity already exists,
+ * it results in an update of the entity.
+ *
+ * This method should be reserved for selected critical entities and events.
+ * For normal voluminous writes one should use the async method
+ * {@link #postEntitiesAsync(TimelineEntities, UserGroupInformation)}.
+ *
+ * @param entities entities to post
+ * @param callerUgi the caller UGI
+ * @return the response that contains the result of the post.
+ */
+ public TimelineWriteResponse postEntities(TimelineEntities entities,
+ UserGroupInformation callerUgi) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SUCCESS - TIMELINE V2 PROTOTYPE");
+ LOG.debug("postEntities(entities=" + entities + ", callerUgi="
+ + callerUgi + ")");
+ }
+
+ return writer.write(entities);
+ }
+
+ /**
+ * Handles entity writes in an asynchronous manner. The method returns as soon
+ * as validation is done. No promises are made on how quickly it will be
+ * written to the backing storage or if it will always be written to the
+ * backing storage. Multiple writes to the same entities may be batched and
+ * appropriate values updated and result in fewer writes to the backing
+ * storage.
+ *
+ * @param entities entities to post
+ * @param callerUgi the caller UGI
+ */
+ public void postEntitiesAsync(TimelineEntities entities,
+ UserGroupInformation callerUgi) {
+ // TODO implement
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("postEntitiesAsync(entities=" + entities + ", callerUgi=" +
+ callerUgi + ")");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
new file mode 100644
index 0000000..3691162
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Class that manages adding and removing collectors and their lifecycle. It
+ * provides thread safety access to the collectors inside.
+ *
+ * It is a singleton, and instances should be obtained via
+ * {@link #getInstance()}.
+ */
+@Private
+@Unstable
+public class TimelineCollectorManager extends CompositeService {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineCollectorManager.class);
+ private static final TimelineCollectorManager INSTANCE =
+ new TimelineCollectorManager();
+
+ // access to this map is synchronized with the map itself
+ private final Map<String, TimelineCollector> collectors =
+ Collections.synchronizedMap(
+ new HashMap<String, TimelineCollector>());
+
+ // REST server for this collector manager
+ private HttpServer2 timelineRestServer;
+
+ private String timelineRestServerBindAddress;
+
+ private CollectorNodemanagerProtocol nmCollectorService;
+
+ private InetSocketAddress nmCollectorServiceAddress;
+
+ static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
+
+ static TimelineCollectorManager getInstance() {
+ return INSTANCE;
+ }
+
+ @VisibleForTesting
+ protected TimelineCollectorManager() {
+ super(TimelineCollectorManager.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ this.nmCollectorServiceAddress = conf.getSocketAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
+
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (timelineRestServer != null) {
+ timelineRestServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ /**
+ * Put the collector into the collection if an collector mapped by id does
+ * not exist.
+ *
+ * @throws YarnRuntimeException if there was any exception in initializing and
+ * starting the app level service
+ * @return the collector associated with id after the potential put.
+ */
+ public TimelineCollector putIfAbsent(ApplicationId appId,
+ TimelineCollector collector) {
+ String id = appId.toString();
+ TimelineCollector collectorInTable;
+ boolean collectorIsNew = false;
+ synchronized (collectors) {
+ collectorInTable = collectors.get(id);
+ if (collectorInTable == null) {
+ try {
+ // initialize, start, and add it to the collection so it can be
+ // cleaned up when the parent shuts down
+ collector.init(getConfig());
+ collector.start();
+ collectors.put(id, collector);
+ LOG.info("the collector for " + id + " was added");
+ collectorInTable = collector;
+ collectorIsNew = true;
+ } catch (Exception e) {
+ throw new YarnRuntimeException(e);
+ }
+ } else {
+ String msg = "the collector for " + id + " already exists!";
+ LOG.error(msg);
+ }
+
+ }
+ // Report to NM if a new collector is added.
+ if (collectorIsNew) {
+ try {
+ reportNewCollectorToNM(appId);
+ } catch (Exception e) {
+ // throw exception here as it cannot be used if failed report to NM
+ LOG.error("Failed to report a new collector for application: " + appId +
+ " to the NM Collector Service.");
+ throw new YarnRuntimeException(e);
+ }
+ }
+
+ return collectorInTable;
+ }
+
+ /**
+ * Removes the collector for the specified id. The collector is also stopped
+ * as a result. If the collector does not exist, no change is made.
+ *
+ * @return whether it was removed successfully
+ */
+ public boolean remove(String id) {
+ synchronized (collectors) {
+ TimelineCollector collector = collectors.remove(id);
+ if (collector == null) {
+ String msg = "the collector for " + id + " does not exist!";
+ LOG.error(msg);
+ return false;
+ } else {
+ // stop the service to do clean up
+ collector.stop();
+ LOG.info("the collector service for " + id + " was removed");
+ return true;
+ }
+ }
+ }
+
+ /**
+ * Returns the collector for the specified id.
+ *
+ * @return the collector or null if it does not exist
+ */
+ public TimelineCollector get(String id) {
+ return collectors.get(id);
+ }
+
+ /**
+ * Returns whether the collector for the specified id exists in this
+ * collection.
+ */
+ public boolean containsKey(String id) {
+ return collectors.containsKey(id);
+ }
+
+ /**
+ * Launch the REST web server for this collector manager
+ */
+ private void startWebApp() {
+ Configuration conf = getConfig();
+ // use the same ports as the old ATS for now; we could create new properties
+ // for the new timeline service if needed
+ String bindAddress = WebAppUtils.getWebAppBindURL(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ WebAppUtils.getAHSWebAppURLWithoutScheme(conf));
+ this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
+ NetUtils.createSocketAddr(bindAddress));
+ LOG.info("Instantiating the per-node collector webapp at " +
+ timelineRestServerBindAddress);
+ try {
+ Configuration confForInfoServer = new Configuration(conf);
+ confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("timeline")
+ .setConf(conf)
+ .addEndpoint(URI.create("http://" + bindAddress));
+ timelineRestServer = builder.build();
+ // TODO: replace this by an authentication filter in future.
+ HashMap<String, String> options = new HashMap<>();
+ String username = conf.get(HADOOP_HTTP_STATIC_USER,
+ DEFAULT_HADOOP_HTTP_STATIC_USER);
+ options.put(HADOOP_HTTP_STATIC_USER, username);
+ HttpServer2.defineFilter(timelineRestServer.getWebAppContext(),
+ "static_user_filter_timeline",
+ StaticUserWebFilter.StaticUserFilter.class.getName(),
+ options, new String[] {"/*"});
+
+ timelineRestServer.addJerseyResourcePackage(
+ TimelineCollectorWebService.class.getPackage().getName() + ";"
+ + GenericExceptionHandler.class.getPackage().getName() + ";"
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
+ "/*");
+ timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY,
+ TimelineCollectorManager.getInstance());
+ timelineRestServer.start();
+ } catch (Exception e) {
+ String msg = "The per-node collector webapp failed to start.";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ }
+
+ private void reportNewCollectorToNM(ApplicationId appId)
+ throws YarnException, IOException {
+ this.nmCollectorService = getNMCollectorService();
+ ReportNewCollectorInfoRequest request =
+ ReportNewCollectorInfoRequest.newInstance(appId,
+ this.timelineRestServerBindAddress);
+ LOG.info("Report a new collector for application: " + appId +
+ " to the NM Collector Service.");
+ nmCollectorService.reportNewCollectorInfo(request);
+ }
+
+ @VisibleForTesting
+ protected CollectorNodemanagerProtocol getNMCollectorService() {
+ Configuration conf = getConfig();
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ // TODO Security settings.
+ return (CollectorNodemanagerProtocol) rpc.getProxy(
+ CollectorNodemanagerProtocol.class,
+ nmCollectorServiceAddress, conf);
+ }
+
+ @VisibleForTesting
+ public String getRestServerBindAddress() {
+ return timelineRestServerBindAddress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
new file mode 100644
index 0000000..5adae71
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -0,0 +1,183 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import javax.servlet.ServletContext;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.PUT;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
+
+import com.google.inject.Singleton;
+
+/**
+ * The main per-node REST end point for timeline service writes. It is
+ * essentially a container service that routes requests to the appropriate
+ * per-app services.
+ */
+@Private
+@Unstable
+@Singleton
+@Path("/ws/v2/timeline")
+public class TimelineCollectorWebService {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineCollectorWebService.class);
+
+ private @Context ServletContext context;
+
+ @XmlRootElement(name = "about")
+ @XmlAccessorType(XmlAccessType.NONE)
+ @Public
+ @Unstable
+ public static class AboutInfo {
+
+ private String about;
+
+ public AboutInfo() {
+
+ }
+
+ public AboutInfo(String about) {
+ this.about = about;
+ }
+
+ @XmlElement(name = "About")
+ public String getAbout() {
+ return about;
+ }
+
+ public void setAbout(String about) {
+ this.about = about;
+ }
+
+ }
+
+ /**
+ * Return the description of the timeline web services.
+ */
+ @GET
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public AboutInfo about(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res) {
+ init(res);
+ return new AboutInfo("Timeline Collector API");
+ }
+
+ /**
+ * Accepts writes to the collector, and returns a response. It simply routes
+ * the request to the app level collector. It expects an application as a
+ * context.
+ */
+ @PUT
+ @Path("/entities")
+ @Consumes({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public Response putEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @QueryParam("async") String async,
+ @QueryParam("appid") String appId,
+ TimelineEntities entities) {
+ init(res);
+ UserGroupInformation callerUgi = getUser(req);
+ if (callerUgi == null) {
+ String msg = "The owner of the posted timeline entities is not set";
+ LOG.error(msg);
+ throw new ForbiddenException(msg);
+ }
+
+ // TODO how to express async posts and handle them
+ boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
+
+ try {
+ appId = parseApplicationId(appId);
+ if (appId == null) {
+ return Response.status(Response.Status.BAD_REQUEST).build();
+ }
+ TimelineCollector collector = getCollector(req, appId);
+ if (collector == null) {
+ LOG.error("Application not found");
+ throw new NotFoundException(); // different exception?
+ }
+ collector.postEntities(entities, callerUgi);
+ return Response.ok().build();
+ } catch (Exception e) {
+ LOG.error("Error putting entities", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ private String parseApplicationId(String appId) {
+ try {
+ if (appId != null) {
+ return ConverterUtils.toApplicationId(appId.trim()).toString();
+ } else {
+ return null;
+ }
+ } catch (Exception e) {
+ return null;
+ }
+ }
+
+ private TimelineCollector
+ getCollector(HttpServletRequest req, String appIdToParse) {
+ String appIdString = parseApplicationId(appIdToParse);
+ final TimelineCollectorManager collectorManager =
+ (TimelineCollectorManager) context.getAttribute(
+ TimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY);
+ return collectorManager.get(appIdString);
+ }
+
+ private void init(HttpServletResponse response) {
+ response.setContentType(null);
+ }
+
+ private UserGroupInformation getUser(HttpServletRequest req) {
+ String remoteUser = req.getRemoteUser();
+ UserGroupInformation callerUgi = null;
+ if (remoteUser != null) {
+ callerUgi = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUgi;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index 4a57e97..f5603f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -65,7 +65,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
* Stores the entire information in {@link TimelineEntity} to the
* timeline store. Any errors occurring for individual write request objects
* will be reported in the response.
- *
+ *
* @param data
* a {@link TimelineEntity} object
* @return {@link TimelineWriteResponse} object.
@@ -116,10 +116,10 @@ public class FileSystemTimelineWriterImpl extends AbstractService
* Aggregates the entity information to the timeline store based on which
* track this entity is to be rolled up to The tracks along which aggregations
* are to be done are given by {@link TimelineAggregationTrack}
- *
+ *
* Any errors occurring for individual write request objects will be reported
* in the response.
- *
+ *
* @param data
* a {@link TimelineEntity} object
* a {@link TimelineAggregationTrack} enum value
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java
deleted file mode 100644
index 8f95814..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestAppLevelTimelineAggregator.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-
-public class TestAppLevelTimelineAggregator {
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java
deleted file mode 100644
index 1c89ead..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestPerNodeTimelineAggregatorsAuxService.java
+++ /dev/null
@@ -1,150 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.util.ExitUtil;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
-import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
-import org.junit.Test;
-
-public class TestPerNodeTimelineAggregatorsAuxService {
- private ApplicationAttemptId appAttemptId;
-
- public TestPerNodeTimelineAggregatorsAuxService() {
- ApplicationId appId =
- ApplicationId.newInstance(System.currentTimeMillis(), 1);
- appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
- }
-
- @Test
- public void testAddApplication() throws Exception {
- PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication();
- // auxService should have a single app
- assertTrue(auxService.hasApplication(
- appAttemptId.getApplicationId().toString()));
- auxService.close();
- }
-
- @Test
- public void testAddApplicationNonAMContainer() throws Exception {
- PerNodeTimelineAggregatorsAuxService auxService = createAggregator();
-
- ContainerId containerId = getContainerId(2L); // not an AM
- ContainerInitializationContext context =
- mock(ContainerInitializationContext.class);
- when(context.getContainerId()).thenReturn(containerId);
- auxService.initializeContainer(context);
- // auxService should not have that app
- assertFalse(auxService.hasApplication(
- appAttemptId.getApplicationId().toString()));
- }
-
- @Test
- public void testRemoveApplication() throws Exception {
- PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication();
- // auxService should have a single app
- String appIdStr = appAttemptId.getApplicationId().toString();
- assertTrue(auxService.hasApplication(appIdStr));
-
- ContainerId containerId = getAMContainerId();
- ContainerTerminationContext context =
- mock(ContainerTerminationContext.class);
- when(context.getContainerId()).thenReturn(containerId);
- auxService.stopContainer(context);
- // auxService should not have that app
- assertFalse(auxService.hasApplication(appIdStr));
- auxService.close();
- }
-
- @Test
- public void testRemoveApplicationNonAMContainer() throws Exception {
- PerNodeTimelineAggregatorsAuxService auxService = createAggregatorAndAddApplication();
- // auxService should have a single app
- String appIdStr = appAttemptId.getApplicationId().toString();
- assertTrue(auxService.hasApplication(appIdStr));
-
- ContainerId containerId = getContainerId(2L); // not an AM
- ContainerTerminationContext context =
- mock(ContainerTerminationContext.class);
- when(context.getContainerId()).thenReturn(containerId);
- auxService.stopContainer(context);
- // auxService should still have that app
- assertTrue(auxService.hasApplication(appIdStr));
- auxService.close();
- }
-
- @Test(timeout = 60000)
- public void testLaunch() throws Exception {
- ExitUtil.disableSystemExit();
- PerNodeTimelineAggregatorsAuxService auxService = null;
- try {
- auxService =
- PerNodeTimelineAggregatorsAuxService.launchServer(new String[0]);
- } catch (ExitUtil.ExitException e) {
- assertEquals(0, e.status);
- ExitUtil.resetFirstExitException();
- fail();
- } finally {
- if (auxService != null) {
- auxService.stop();
- }
- }
- }
-
- private PerNodeTimelineAggregatorsAuxService createAggregatorAndAddApplication() {
- PerNodeTimelineAggregatorsAuxService auxService = createAggregator();
- // create an AM container
- ContainerId containerId = getAMContainerId();
- ContainerInitializationContext context =
- mock(ContainerInitializationContext.class);
- when(context.getContainerId()).thenReturn(containerId);
- auxService.initializeContainer(context);
- return auxService;
- }
-
- private PerNodeTimelineAggregatorsAuxService createAggregator() {
- TimelineAggregatorsCollection
- aggregatorsCollection = spy(new TimelineAggregatorsCollection());
- doReturn(new Configuration()).when(aggregatorsCollection).getConfig();
- PerNodeTimelineAggregatorsAuxService auxService =
- spy(new PerNodeTimelineAggregatorsAuxService(aggregatorsCollection));
- return auxService;
- }
-
- private ContainerId getAMContainerId() {
- return getContainerId(1L);
- }
-
- private ContainerId getContainerId(long id) {
- return ContainerId.newContainerId(appAttemptId, id);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java
deleted file mode 100644
index dd64629..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/aggregator/TestTimelineAggregatorsCollection.java
+++ /dev/null
@@ -1,109 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.aggregator;
-
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import com.sun.jersey.core.impl.provider.entity.XMLJAXBElementProvider;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.junit.Test;
-
-public class TestTimelineAggregatorsCollection {
-
- @Test(timeout=60000)
- public void testMultithreadedAdd() throws Exception {
- final TimelineAggregatorsCollection aggregatorCollection =
- spy(new TimelineAggregatorsCollection());
- doReturn(new Configuration()).when(aggregatorCollection).getConfig();
-
- final int NUM_APPS = 5;
- List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
- for (int i = 0; i < NUM_APPS; i++) {
- final ApplicationId appId = ApplicationId.newInstance(0L, i);
- Callable<Boolean> task = new Callable<Boolean>() {
- public Boolean call() {
- AppLevelTimelineAggregator aggregator =
- new AppLevelTimelineAggregator(appId.toString());
- return (aggregatorCollection.putIfAbsent(appId, aggregator) == aggregator);
- }
- };
- tasks.add(task);
- }
- ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
- try {
- List<Future<Boolean>> futures = executor.invokeAll(tasks);
- for (Future<Boolean> future: futures) {
- assertTrue(future.get());
- }
- } finally {
- executor.shutdownNow();
- }
- // check the keys
- for (int i = 0; i < NUM_APPS; i++) {
- assertTrue(aggregatorCollection.containsKey(String.valueOf(i)));
- }
- }
-
- @Test
- public void testMultithreadedAddAndRemove() throws Exception {
- final TimelineAggregatorsCollection aggregatorCollection =
- spy(new TimelineAggregatorsCollection());
- doReturn(new Configuration()).when(aggregatorCollection).getConfig();
-
- final int NUM_APPS = 5;
- List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
- for (int i = 0; i < NUM_APPS; i++) {
- final ApplicationId appId = ApplicationId.newInstance(0L, i);
- Callable<Boolean> task = new Callable<Boolean>() {
- public Boolean call() {
- AppLevelTimelineAggregator aggregator =
- new AppLevelTimelineAggregator(appId.toString());
- boolean successPut =
- (aggregatorCollection.putIfAbsent(appId, aggregator) == aggregator);
- return successPut && aggregatorCollection.remove(appId.toString());
- }
- };
- tasks.add(task);
- }
- ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
- try {
- List<Future<Boolean>> futures = executor.invokeAll(tasks);
- for (Future<Boolean> future: futures) {
- assertTrue(future.get());
- }
- } finally {
- executor.shutdownNow();
- }
- // check the keys
- for (int i = 0; i < NUM_APPS; i++) {
- assertFalse(aggregatorCollection.containsKey(String.valueOf(i)));
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
new file mode 100644
index 0000000..74c81a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestAppLevelTimelineCollector.java
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+
+public class TestAppLevelTimelineCollector {
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
new file mode 100644
index 0000000..3b20352
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
+import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.junit.Test;
+
+public class TestPerNodeTimelineCollectorsAuxService {
+ private ApplicationAttemptId appAttemptId;
+
+ public TestPerNodeTimelineCollectorsAuxService() {
+ ApplicationId appId =
+ ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ }
+
+ @Test
+ public void testAddApplication() throws Exception {
+ PerNodeTimelineCollectorsAuxService auxService =
+ createCollectorAndAddApplication();
+ // auxService should have a single app
+ assertTrue(auxService.hasApplication(
+ appAttemptId.getApplicationId().toString()));
+ auxService.close();
+ }
+
+ @Test
+ public void testAddApplicationNonAMContainer() throws Exception {
+ PerNodeTimelineCollectorsAuxService auxService = createCollector();
+
+ ContainerId containerId = getContainerId(2L); // not an AM
+ ContainerInitializationContext context =
+ mock(ContainerInitializationContext.class);
+ when(context.getContainerId()).thenReturn(containerId);
+ auxService.initializeContainer(context);
+ // auxService should not have that app
+ assertFalse(auxService.hasApplication(
+ appAttemptId.getApplicationId().toString()));
+ }
+
+ @Test
+ public void testRemoveApplication() throws Exception {
+ PerNodeTimelineCollectorsAuxService auxService =
+ createCollectorAndAddApplication();
+ // auxService should have a single app
+ String appIdStr = appAttemptId.getApplicationId().toString();
+ assertTrue(auxService.hasApplication(appIdStr));
+
+ ContainerId containerId = getAMContainerId();
+ ContainerTerminationContext context =
+ mock(ContainerTerminationContext.class);
+ when(context.getContainerId()).thenReturn(containerId);
+ auxService.stopContainer(context);
+ // auxService should not have that app
+ assertFalse(auxService.hasApplication(appIdStr));
+ auxService.close();
+ }
+
+ @Test
+ public void testRemoveApplicationNonAMContainer() throws Exception {
+ PerNodeTimelineCollectorsAuxService auxService =
+ createCollectorAndAddApplication();
+ // auxService should have a single app
+ String appIdStr = appAttemptId.getApplicationId().toString();
+ assertTrue(auxService.hasApplication(appIdStr));
+
+ ContainerId containerId = getContainerId(2L); // not an AM
+ ContainerTerminationContext context =
+ mock(ContainerTerminationContext.class);
+ when(context.getContainerId()).thenReturn(containerId);
+ auxService.stopContainer(context);
+ // auxService should still have that app
+ assertTrue(auxService.hasApplication(appIdStr));
+ auxService.close();
+ }
+
+ @Test(timeout = 60000)
+ public void testLaunch() throws Exception {
+ ExitUtil.disableSystemExit();
+ PerNodeTimelineCollectorsAuxService auxService = null;
+ try {
+ auxService =
+ PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
+ createCollectorManager());
+ } catch (ExitUtil.ExitException e) {
+ assertEquals(0, e.status);
+ ExitUtil.resetFirstExitException();
+ fail();
+ } finally {
+ if (auxService != null) {
+ auxService.stop();
+ }
+ }
+ }
+
+ private PerNodeTimelineCollectorsAuxService
+ createCollectorAndAddApplication() {
+ PerNodeTimelineCollectorsAuxService auxService = createCollector();
+ // create an AM container
+ ContainerId containerId = getAMContainerId();
+ ContainerInitializationContext context =
+ mock(ContainerInitializationContext.class);
+ when(context.getContainerId()).thenReturn(containerId);
+ auxService.initializeContainer(context);
+ return auxService;
+ }
+
+ private PerNodeTimelineCollectorsAuxService createCollector() {
+ TimelineCollectorManager collectorManager = createCollectorManager();
+ PerNodeTimelineCollectorsAuxService auxService =
+ spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
+ return auxService;
+ }
+
+ private TimelineCollectorManager createCollectorManager() {
+ TimelineCollectorManager collectorManager =
+ spy(new TimelineCollectorManager());
+ doReturn(new Configuration()).when(collectorManager).getConfig();
+ CollectorNodemanagerProtocol nmCollectorService =
+ mock(CollectorNodemanagerProtocol.class);
+ doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
+ return collectorManager;
+ }
+
+ private ContainerId getAMContainerId() {
+ return getContainerId(1L);
+ }
+
+ private ContainerId getContainerId(long id) {
+ return ContainerId.newContainerId(appAttemptId, id);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
new file mode 100644
index 0000000..541665b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.junit.Test;
+
+public class TestTimelineCollectorManager {
+
+ @Test(timeout=60000)
+ public void testMultithreadedAdd() throws Exception {
+ final TimelineCollectorManager collectorManager = createCollectorManager();
+
+ final int NUM_APPS = 5;
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ Callable<Boolean> task = new Callable<Boolean>() {
+ public Boolean call() {
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId.toString());
+ return (collectorManager.putIfAbsent(appId, collector) == collector);
+ }
+ };
+ tasks.add(task);
+ }
+ ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
+ try {
+ List<Future<Boolean>> futures = executor.invokeAll(tasks);
+ for (Future<Boolean> future: futures) {
+ assertTrue(future.get());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ // check the keys
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ assertTrue(collectorManager.containsKey(appId.toString()));
+ }
+ }
+
+ @Test
+ public void testMultithreadedAddAndRemove() throws Exception {
+ final TimelineCollectorManager collectorManager = createCollectorManager();
+
+ final int NUM_APPS = 5;
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ Callable<Boolean> task = new Callable<Boolean>() {
+ public Boolean call() {
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId.toString());
+ boolean successPut =
+ (collectorManager.putIfAbsent(appId, collector) == collector);
+ return successPut && collectorManager.remove(appId.toString());
+ }
+ };
+ tasks.add(task);
+ }
+ ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
+ try {
+ List<Future<Boolean>> futures = executor.invokeAll(tasks);
+ for (Future<Boolean> future: futures) {
+ assertTrue(future.get());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ // check the keys
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ assertFalse(collectorManager.containsKey(appId.toString()));
+ }
+ }
+
+ private TimelineCollectorManager createCollectorManager() {
+ final TimelineCollectorManager collectorManager =
+ spy(new TimelineCollectorManager());
+ doReturn(new Configuration()).when(collectorManager).getConfig();
+ CollectorNodemanagerProtocol nmCollectorService =
+ mock(CollectorNodemanagerProtocol.class);
+ doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
+ return collectorManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63c7210c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index f720454..7f919f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -27,12 +27,12 @@ import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
-import org.apache.commons.io.FileUtils;
public class TestFileSystemTimelineWriterImpl {
@@ -42,9 +42,6 @@ public class TestFileSystemTimelineWriterImpl {
*/
@Test
public void testWriteEntityToFile() throws Exception {
- String name = "unit_test_BaseAggregator_testWriteEntityToFile_"
- + Long.toString(System.currentTimeMillis());
-
TimelineEntities te = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
String id = "hello";
@@ -55,25 +52,27 @@ public class TestFileSystemTimelineWriterImpl {
entity.setModifiedTime(1425016502000L);
te.addEntity(entity);
- FileSystemTimelineWriterImpl fsi = new FileSystemTimelineWriterImpl();
- fsi.serviceInit(new Configuration());
- fsi.write(te);
+ try (FileSystemTimelineWriterImpl fsi =
+ new FileSystemTimelineWriterImpl()) {
+ fsi.serviceInit(new Configuration());
+ fsi.write(te);
- String fileName = fsi.getOutputRoot() + "/" + type + "/" + id
- + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
- Path path = Paths.get(fileName);
- File f = new File(fileName);
- assertTrue(f.exists() && !f.isDirectory());
- List<String> data = Files.readAllLines(path, StandardCharsets.UTF_8);
- // ensure there's only one entity + 1 new line
- assertTrue(data.size() == 2);
- String d = data.get(0);
- // confirm the contents same as what was written
- assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ String fileName = fsi.getOutputRoot() + "/" + type + "/" + id
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ Path path = Paths.get(fileName);
+ File f = new File(fileName);
+ assertTrue(f.exists() && !f.isDirectory());
+ List<String> data = Files.readAllLines(path, StandardCharsets.UTF_8);
+ // ensure there's only one entity + 1 new line
+ assertTrue(data.size() == 2);
+ String d = data.get(0);
+ // confirm the contents same as what was written
+ assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
- // delete the directory
- File outputDir = new File(fsi.getOutputRoot());
- FileUtils.deleteDirectory(outputDir);
- assertTrue(!(f.exists()));
+ // delete the directory
+ File outputDir = new File(fsi.getOutputRoot());
+ FileUtils.deleteDirectory(outputDir);
+ assertTrue(!(f.exists()));
+ }
}
}
[30/50] [abbrv] hadoop git commit: YARN-3801. [JDK-8] Exclude
jdk.tools from hbase-client and hbase-testing-util (Tsuyoshi Ozawa via sjlee)
Posted by sj...@apache.org.
YARN-3801. [JDK-8] Exclude jdk.tools from hbase-client and hbase-testing-util (Tsuyoshi Ozawa via sjlee)
(cherry picked from commit a1bb9137af84a34bde799f45e7ab8a21e33d55e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/623b3abc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/623b3abc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/623b3abc
Branch: refs/heads/YARN-2928
Commit: 623b3abc6789da026657b18ee6de6ae106e2af37
Parents: a22d26c
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Jun 15 21:15:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:13 2015 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 10 ++++++++++
hadoop-yarn-project/CHANGES.txt | 3 +++
2 files changed, 13 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/623b3abc/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 1182d6d..faac713 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1000,6 +1000,12 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-client</artifactId>
<version>${hbase.version}</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>jdk.tools</artifactId>
+ <groupId>jdk.tools</groupId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.phoenix</groupId>
@@ -1046,6 +1052,10 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
</exclusion>
+ <exclusion>
+ <artifactId>jdk.tools</artifactId>
+ <groupId>jdk.tools</groupId>
+ </exclusion>
</exclusions>
</dependency>
</dependencies>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/623b3abc/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a673fb2..1cf1f26 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -85,6 +85,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3044. Made RM write app, attempt and optional container lifecycle
events to timeline service v2. (Naganarasimha G R via zjshen)
+ YARN-3801. [JDK-8] Exclude jdk.tools from hbase-client and
+ hbase-testing-util (Tsuyoshi Ozawa via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
[18/50] [abbrv] hadoop git commit: YARN-3634.
TestMRTimelineEventHandling and TestApplication are broken. Contributed by
Sangjin Lee.
Posted by sj...@apache.org.
YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. Contributed by Sangjin Lee.
(cherry picked from commit b059dd4882fd759e4762cc11c019be4b68fb74c1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e93fa602
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e93fa602
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e93fa602
Branch: refs/heads/YARN-2928
Commit: e93fa6023bf6c154aa4817a45020d385fefed22a
Parents: 7a68cda
Author: Junping Du <ju...@apache.org>
Authored: Wed May 13 11:54:24 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:11 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../collectormanager/NMCollectorService.java | 5 +++
.../containermanager/ContainerManagerImpl.java | 2 +-
.../application/TestApplication.java | 3 +-
.../collector/NodeTimelineCollectorManager.java | 46 ++++++++++----------
5 files changed, 34 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e93fa602/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0b06502..ec9abc9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -70,6 +70,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3529. Added mini HBase cluster and Phoenix support to timeline service
v2 unit tests. (Li Lu via zjshen)
+ YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
+ Sangjin Lee via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e93fa602/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index dc5601f..db79ee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -81,6 +81,11 @@ public class NMCollectorService extends CompositeService implements
YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_THREAD_COUNT));
server.start();
+ collectorServerAddress = conf.updateConnectAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ server.getListenerAddress());
// start remaining services
super.serviceStart();
LOG.info("NMCollectorService started at " + collectorServerAddress);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e93fa602/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 4dd9fa6..aa9b102 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -878,7 +878,7 @@ public class ContainerManagerImpl extends CompositeService implements
TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
long flowRunId = 0L;
if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
- flowRunId = Long.valueOf(flowRunIdStr);
+ flowRunId = Long.parseLong(flowRunIdStr);
}
Application application = new ApplicationImpl(dispatcher, user,
flowName, flowVersion, flowRunId, applicationID, credentials, context);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e93fa602/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 3889d2e..002d4cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -519,7 +519,8 @@ public class TestApplication {
when(context.getApplicationACLsManager()).thenReturn(
new ApplicationACLsManager(conf));
when(context.getNMTokenSecretManager()).thenReturn(nmTokenSecretMgr);
-
+ when(context.getConf()).thenReturn(conf);
+
// Setting master key
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.setKeyId(123);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e93fa602/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 03ac770..31051db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -69,9 +69,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
private String timelineRestServerBindAddress;
- private CollectorNodemanagerProtocol nmCollectorService;
-
- private InetSocketAddress nmCollectorServiceAddress;
+ private volatile CollectorNodemanagerProtocol nmCollectorService;
static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
@@ -85,18 +83,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
}
@Override
- public void serviceInit(Configuration conf) throws Exception {
- this.nmCollectorServiceAddress = conf.getSocketAddr(
- YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
- super.serviceInit(conf);
- }
-
- @Override
protected void serviceStart() throws Exception {
- nmCollectorService = getNMCollectorService();
startWebApp();
super.serviceStart();
}
@@ -176,7 +163,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
this.timelineRestServerBindAddress);
LOG.info("Report a new collector for application: " + appId +
" to the NM Collector Service.");
- nmCollectorService.reportNewCollectorInfo(request);
+ getNMCollectorService().reportNewCollectorInfo(request);
}
private void updateTimelineCollectorContext(
@@ -186,7 +173,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
GetTimelineCollectorContextRequest.newInstance(appId);
LOG.info("Get timeline collector context for " + appId);
GetTimelineCollectorContextResponse response =
- nmCollectorService.getTimelineCollectorContext(request);
+ getNMCollectorService().getTimelineCollectorContext(request);
String userId = response.getUserId();
if (userId != null && !userId.isEmpty()) {
collector.getTimelineEntityContext().setUserId(userId);
@@ -207,13 +194,26 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
@VisibleForTesting
protected CollectorNodemanagerProtocol getNMCollectorService() {
- Configuration conf = getConfig();
- final YarnRPC rpc = YarnRPC.create(conf);
-
- // TODO Security settings.
- return (CollectorNodemanagerProtocol) rpc.getProxy(
- CollectorNodemanagerProtocol.class,
- nmCollectorServiceAddress, conf);
+ if (nmCollectorService == null) {
+ synchronized (this) {
+ if (nmCollectorService == null) {
+ Configuration conf = getConfig();
+ InetSocketAddress nmCollectorServiceAddress = conf.getSocketAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
+ LOG.info("nmCollectorServiceAddress: " + nmCollectorServiceAddress);
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ // TODO Security settings.
+ nmCollectorService = (CollectorNodemanagerProtocol) rpc.getProxy(
+ CollectorNodemanagerProtocol.class,
+ nmCollectorServiceAddress, conf);
+ }
+ }
+ }
+ return nmCollectorService;
}
@VisibleForTesting
[46/50] [abbrv] hadoop git commit: YARN-3045. Implement NM writing
container lifecycle events to Timeline Service v2. Contributed by
Naganarasimha G R.
Posted by sj...@apache.org.
YARN-3045. Implement NM writing container lifecycle events to Timeline Service v2. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/702a2142
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/702a2142
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/702a2142
Branch: refs/heads/YARN-2928
Commit: 702a2142b15c45f12aa79c28c5148984e0452ed3
Parents: 97f211b
Author: Junping Du <ju...@apache.org>
Authored: Tue Aug 18 04:31:45 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:52:44 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 16 +-
.../distributedshell/TestDistributedShell.java | 28 +-
.../hadoop/yarn/server/nodemanager/Context.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 13 +
.../containermanager/ContainerManagerImpl.java | 48 ++-
.../ApplicationContainerFinishedEvent.java | 17 +-
.../containermanager/container/Container.java | 3 +
.../container/ContainerImpl.java | 29 +-
.../monitor/ContainersMonitorImpl.java | 108 +-----
.../timelineservice/NMTimelineEvent.java | 31 ++
.../timelineservice/NMTimelineEventType.java | 24 ++
.../timelineservice/NMTimelinePublisher.java | 376 +++++++++++++++++++
.../nodemanager/TestNodeStatusUpdater.java | 24 +-
.../containermanager/TestAuxServices.java | 4 +-
.../TestContainerManagerRecovery.java | 8 +
.../application/TestApplication.java | 6 +-
.../container/TestContainer.java | 2 +-
.../nodemanager/webapp/MockContainer.java | 6 +
.../nodemanager/webapp/TestNMWebServer.java | 3 +-
.../PerNodeTimelineCollectorsAuxService.java | 16 +-
...TestPerNodeTimelineCollectorsAuxService.java | 9 +
22 files changed, 636 insertions(+), 143 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3e1f212..492a098 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -91,6 +91,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3904. Refactor timelineservice.storage to add support to online and
offline aggregation writers (Li Lu via sjlee)
+ YARN-3045. Implement NM writing container lifecycle events to Timeline
+ Service v2. (Naganarasimha G R via junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 691170e..62e60a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -110,6 +110,16 @@
<Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
+ <!-- Object cast is based on the event type -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher$ApplicationEventHandler" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
<!-- Ignore intentional switch fallthroughs -->
<Match>
@@ -505,10 +515,4 @@
</Or>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
-
- <!-- Object cast is based on the event type -->
- <Match>
- <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
- <Bug pattern="BC_UNCONFIRMED_CAST" />
- </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index b8a7abf..c89bee9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
@@ -203,7 +204,7 @@ public class TestDistributedShell {
testDSShell(false, "v2", false);
}
- public void testDSShell(boolean haveDomain, String timelineVersion,
+ private void testDSShell(boolean haveDomain, String timelineVersion,
boolean defaultFlow)
throws Exception {
String[] args = {
@@ -404,9 +405,32 @@ public class TestDistributedShell {
"container_" + appId.getClusterTimestamp() + "_000" + appId.getId()
+ "_01_000001"
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
- verifyEntityTypeFileExists(basePath,
+ File containerEntityFile = verifyEntityTypeFileExists(basePath,
TimelineEntityType.YARN_CONTAINER.toString(),
containerMetricsTimestampFileName);
+ Assert.assertEquals(
+ "Container created event needs to be published atleast once",
+ 1,
+ getNumOfStringOccurences(containerEntityFile,
+ ContainerMetricsConstants.CREATED_EVENT_TYPE));
+
+ // to avoid race condition of testcase, atleast check 4 times with sleep
+ // of 500ms
+ long numOfContainerFinishedOccurences = 0;
+ for (int i = 0; i < 4; i++) {
+ numOfContainerFinishedOccurences =
+ getNumOfStringOccurences(containerEntityFile,
+ ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ if (numOfContainerFinishedOccurences > 0) {
+ break;
+ } else {
+ Thread.sleep(500l);
+ }
+ }
+ Assert.assertEquals(
+ "Container finished event needs to be published atleast once",
+ 1,
+ numOfContainerFinishedOccurences);
// Verify RM posting Application life cycle Events are getting published
String appMetricsTimestampFileName =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 179dd92..4999b7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
/**
@@ -93,4 +94,8 @@ public interface Context {
ConcurrentLinkedQueue<LogAggregationReport>
getLogAggregationStatusForApps();
+
+ void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher);
+
+ NMTimelinePublisher getNMTimelinePublisher();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 697b706..4b79091 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreServic
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -468,6 +469,8 @@ public class NodeManager extends CompositeService
private final ConcurrentLinkedQueue<LogAggregationReport>
logAggregationReportForApps;
+ private NMTimelinePublisher nmTimelinePublisher;
+
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
@@ -604,6 +607,16 @@ public class NodeManager extends CompositeService
Map<ApplicationId, String> newRegisteredCollectors) {
this.registeredCollectors.putAll(newRegisteredCollectors);
}
+
+ @Override
+ public void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher) {
+ this.nmTimelinePublisher = nmMetricsPublisher;
+ }
+
+ @Override
+ public NMTimelinePublisher getNMTimelinePublisher() {
+ return nmTimelinePublisher;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index aa9b102..2ea2ec1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -118,6 +118,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadService;
@@ -134,12 +135,13 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.Re
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
public class ContainerManagerImpl extends CompositeService implements
ServiceStateChangeListener, ContainerManagementProtocol,
@@ -178,6 +180,8 @@ public class ContainerManagerImpl extends CompositeService implements
private long waitForContainersOnShutdownMillis;
+ private final NMTimelinePublisher nmMetricsPublisher;
+
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
@@ -206,6 +210,8 @@ public class ContainerManagerImpl extends CompositeService implements
auxiliaryServices.registerServiceListener(this);
addService(auxiliaryServices);
+ nmMetricsPublisher = createNMTimelinePublisher(context);
+ context.setNMTimelinePublisher(nmMetricsPublisher);
this.containersMonitor =
new ContainersMonitorImpl(exec, dispatcher, this.context);
addService(this.containersMonitor);
@@ -214,13 +220,16 @@ public class ContainerManagerImpl extends CompositeService implements
new ContainerEventDispatcher());
dispatcher.register(ApplicationEventType.class,
new ApplicationEventDispatcher());
- dispatcher.register(LocalizationEventType.class, rsrcLocalizationSrvc);
+ dispatcher.register(LocalizationEventType.class,
+ new LocalizationEventHandlerWrapper(rsrcLocalizationSrvc,
+ nmMetricsPublisher));
dispatcher.register(AuxServicesEventType.class, auxiliaryServices);
dispatcher.register(ContainersMonitorEventType.class, containersMonitor);
dispatcher.register(ContainersLauncherEventType.class, containersLauncher);
addService(dispatcher);
+
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
@@ -324,7 +333,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container = new ContainerImpl(getConfig(), dispatcher,
context.getNMStateStore(), req.getContainerLaunchContext(),
credentials, metrics, token, rcs.getStatus(), rcs.getExitCode(),
- rcs.getDiagnostics(), rcs.getKilled());
+ rcs.getDiagnostics(), rcs.getKilled(), context);
context.getContainers().put(containerId, container);
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
@@ -386,6 +395,13 @@ public class ContainerManagerImpl extends CompositeService implements
return new SharedCacheUploadService();
}
+ @VisibleForTesting
+ protected NMTimelinePublisher createNMTimelinePublisher(Context context) {
+ NMTimelinePublisher nmTimelinePublisherLocal = new NMTimelinePublisher(context);
+ addIfService(nmTimelinePublisherLocal);
+ return nmTimelinePublisherLocal;
+ }
+
protected ContainersLauncher createContainersLauncher(Context context,
ContainerExecutor exec) {
return new ContainersLauncher(context, this.dispatcher, exec, dirsHandler, this);
@@ -855,7 +871,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container container =
new ContainerImpl(getConfig(), this.dispatcher,
context.getNMStateStore(), launchContext,
- credentials, metrics, containerTokenIdentifier);
+ credentials, metrics, containerTokenIdentifier, context);
ApplicationId applicationID =
containerId.getApplicationAttemptId().getApplicationId();
if (context.getContainers().putIfAbsent(containerId, container) != null) {
@@ -897,9 +913,9 @@ public class ContainerManagerImpl extends CompositeService implements
logAggregationContext));
}
- this.context.getNMStateStore().storeContainer(containerId, request);
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
+ this.context.getNMStateStore().storeContainer(containerId, request);
this.context.getContainerTokenSecretManager().startContainerSuccessful(
containerTokenIdentifier);
@@ -1116,6 +1132,7 @@ public class ContainerManagerImpl extends CompositeService implements
Container c = containers.get(event.getContainerID());
if (c != null) {
c.handle(event);
+ nmMetricsPublisher.publishContainerEvent(event);
} else {
LOG.warn("Event " + event + " sent to absent container " +
event.getContainerID());
@@ -1124,7 +1141,6 @@ public class ContainerManagerImpl extends CompositeService implements
}
class ApplicationEventDispatcher implements EventHandler<ApplicationEvent> {
-
@Override
public void handle(ApplicationEvent event) {
Application app =
@@ -1132,6 +1148,7 @@ public class ContainerManagerImpl extends CompositeService implements
event.getApplicationID());
if (app != null) {
app.handle(event);
+ nmMetricsPublisher.publishApplicationEvent(event);
} else {
LOG.warn("Event " + event + " sent to absent application "
+ event.getApplicationID());
@@ -1139,6 +1156,25 @@ public class ContainerManagerImpl extends CompositeService implements
}
}
+ private static final class LocalizationEventHandlerWrapper implements
+ EventHandler<LocalizationEvent> {
+
+ private EventHandler<LocalizationEvent> origLocalizationEventHandler;
+ private NMTimelinePublisher timelinePublisher;
+
+ LocalizationEventHandlerWrapper(EventHandler<LocalizationEvent> handler,
+ NMTimelinePublisher publisher) {
+ this.origLocalizationEventHandler = handler;
+ this.timelinePublisher = publisher;
+ }
+
+ @Override
+ public void handle(LocalizationEvent event) {
+ origLocalizationEventHandler.handle(event);
+ timelinePublisher.publishLocalizationEvent(event);
+ }
+ }
+
@SuppressWarnings("unchecked")
@Override
public void handle(ContainerManagerEvent event) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
index 6b8007f..9cd34cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationContainerFinishedEvent.java
@@ -19,18 +19,23 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.application;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
public class ApplicationContainerFinishedEvent extends ApplicationEvent {
- private ContainerId containerID;
+ private ContainerStatus containerStatus;
- public ApplicationContainerFinishedEvent(
- ContainerId containerID) {
- super(containerID.getApplicationAttemptId().getApplicationId(),
+ public ApplicationContainerFinishedEvent(ContainerStatus containerStatus) {
+ super(containerStatus.getContainerId().getApplicationAttemptId().getApplicationId(),
ApplicationEventType.APPLICATION_CONTAINER_FINISHED);
- this.containerID = containerID;
+ this.containerStatus = containerStatus;
}
public ContainerId getContainerID() {
- return this.containerID;
+ return containerStatus.getContainerId();
}
+
+ public ContainerStatus getContainerStatus() {
+ return containerStatus;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 56b4fdd..cc92573 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
@@ -55,4 +56,6 @@ public interface Container extends EventHandler<ContainerEvent> {
String toString();
+ Priority getPriority();
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 3c76596..b934175 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -44,12 +44,14 @@ import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger;
import org.apache.hadoop.yarn.server.nodemanager.NMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -70,6 +72,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
@@ -124,11 +127,12 @@ public class ContainerImpl implements Container {
RecoveredContainerStatus.REQUESTED;
// whether container was marked as killed after recovery
private boolean recoveredAsKilled = false;
+ private Context context;
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
NMStateStoreService stateStore, ContainerLaunchContext launchContext,
Credentials creds, NodeManagerMetrics metrics,
- ContainerTokenIdentifier containerTokenIdentifier) {
+ ContainerTokenIdentifier containerTokenIdentifier, Context context) {
this.daemonConf = conf;
this.dispatcher = dispatcher;
this.stateStore = stateStore;
@@ -143,8 +147,8 @@ public class ContainerImpl implements Container {
ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
this.readLock = readWriteLock.readLock();
this.writeLock = readWriteLock.writeLock();
-
stateMachine = stateMachineFactory.make(this);
+ this.context = context;
}
// constructor for a recovered container
@@ -153,9 +157,9 @@ public class ContainerImpl implements Container {
Credentials creds, NodeManagerMetrics metrics,
ContainerTokenIdentifier containerTokenIdentifier,
RecoveredContainerStatus recoveredStatus, int exitCode,
- String diagnostics, boolean wasKilled) {
+ String diagnostics, boolean wasKilled, Context context) {
this(conf, dispatcher, stateStore, launchContext, creds, metrics,
- containerTokenIdentifier);
+ containerTokenIdentifier, context);
this.recoveredStatus = recoveredStatus;
this.exitCode = exitCode;
this.recoveredAsKilled = wasKilled;
@@ -365,6 +369,10 @@ public class ContainerImpl implements Container {
}
}
+ public NMTimelinePublisher getNMTimelinePublisher() {
+ return context.getNMTimelinePublisher();
+ }
+
@Override
public String getUser() {
this.readLock.lock();
@@ -469,7 +477,10 @@ public class ContainerImpl implements Container {
// Inform the application
@SuppressWarnings("rawtypes")
EventHandler eventHandler = dispatcher.getEventHandler();
- eventHandler.handle(new ApplicationContainerFinishedEvent(containerId));
+
+ ContainerStatus containerStatus = cloneAndGetContainerStatus();
+ eventHandler.handle(new ApplicationContainerFinishedEvent(containerStatus));
+
// Remove the container from the resource-monitor
eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
// Tell the logService too
@@ -971,7 +982,8 @@ public class ContainerImpl implements Container {
public void transition(ContainerImpl container, ContainerEvent event) {
container.metrics.releaseContainer(container.resource);
container.sendFinishedEvents();
- //if the current state is NEW it means the CONTAINER_INIT was never
+
+ // if the current state is NEW it means the CONTAINER_INIT was never
// sent for the event, thus no need to send the CONTAINER_STOP
if (container.getCurrentState()
!= org.apache.hadoop.yarn.api.records.ContainerState.NEW) {
@@ -1157,4 +1169,9 @@ public class ContainerImpl implements Container {
LocalResourceRequest resource) {
return container.resourcesUploadPolicies.get(resource);
}
+
+ @Override
+ public Priority getPriority() {
+ return containerTokenIdentifier.getPriority();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 79c1907..c768db9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -18,16 +18,12 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
-import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -37,24 +33,19 @@ import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import com.google.common.base.Preconditions;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainersMonitorImpl extends AbstractService implements
ContainersMonitor {
@@ -85,17 +76,11 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean pmemCheckEnabled;
private boolean vmemCheckEnabled;
private boolean containersMonitorEnabled;
-
- private boolean publishContainerMetricsToTimelineService;
private long maxVCoresAllottedForContainers;
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
private int nodeCpuPercentageForYARN;
-
- // For posting entities in new timeline service in a non-blocking way
- // TODO replace with event loop in TimelineClient.
- private static ExecutorService threadPool;
@Private
public static enum ContainerMetric {
@@ -206,22 +191,6 @@ public class ContainersMonitorImpl extends AbstractService implements
1) + "). Thrashing might happen.");
}
}
-
- publishContainerMetricsToTimelineService =
- YarnConfiguration.systemMetricsPublisherEnabled(conf);
-
- if (publishContainerMetricsToTimelineService) {
- LOG.info("NodeManager has been configured to publish container " +
- "metrics to Timeline Service V2.");
- threadPool =
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
- } else {
- LOG.warn("NodeManager has not been configured to publish container " +
- "metrics to Timeline Service V2.");
- }
-
super.serviceInit(conf);
}
@@ -264,30 +233,9 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
- shutdownAndAwaitTermination();
-
super.serviceStop();
}
- // TODO remove threadPool after adding non-blocking call in TimelineClient
- private static void shutdownAndAwaitTermination() {
- if (threadPool == null) {
- return;
- }
- threadPool.shutdown();
- try {
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
- threadPool.shutdownNow();
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
- LOG.error("ThreadPool did not terminate");
- }
- } catch (InterruptedException ie) {
- threadPool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
private static class ProcessTreeInfo {
private ContainerId containerId;
private String pid;
@@ -475,9 +423,6 @@ public class ContainersMonitorImpl extends AbstractService implements
ContainerId containerId = entry.getKey();
ProcessTreeInfo ptInfo = entry.getValue();
- ContainerEntity entity = new ContainerEntity();
- entity.setId(containerId.toString());
-
try {
String pId = ptInfo.getPID();
@@ -563,26 +508,6 @@ public class ContainersMonitorImpl extends AbstractService implements
containerId, containerMetricsPeriodMs).recordCpuUsage
((int)cpuUsagePercentPerCore, milliVcoresUsed);
}
-
- if (publishContainerMetricsToTimelineService) {
- // if currentPmemUsage data is available
- if (currentPmemUsage !=
- ResourceCalculatorProcessTree.UNAVAILABLE) {
- TimelineMetric memoryMetric = new TimelineMetric();
- memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
- memoryMetric.addValue(currentTime, currentPmemUsage);
- entity.addMetric(memoryMetric);
- }
- // if cpuUsageTotalCoresPercentage data is available
- if (cpuUsageTotalCoresPercentage !=
- ResourceCalculatorProcessTree.UNAVAILABLE) {
- TimelineMetric cpuMetric = new TimelineMetric();
- cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
- cpuMetric.addValue(currentTime,
- cpuUsageTotalCoresPercentage);
- entity.addMetric(cpuMetric);
- }
- }
boolean isMemoryOverLimit = false;
String msg = "";
@@ -639,23 +564,16 @@ public class ContainersMonitorImpl extends AbstractService implements
LOG.info("Removed ProcessTree with root " + pId);
}
+ ContainerImpl container =
+ (ContainerImpl) context.getContainers().get(containerId);
+ container.getNMTimelinePublisher().reportContainerResourceUsage(
+ container, currentTime, pId, currentPmemUsage,
+ cpuUsageTotalCoresPercentage);
} catch (Exception e) {
// Log the exception and proceed to the next container.
LOG.warn("Uncaught exception in ContainersMonitorImpl "
+ "while monitoring resource of " + containerId, e);
}
-
- if (publishContainerMetricsToTimelineService) {
- try {
- TimelineClient timelineClient = context.getApplications().get(
- containerId.getApplicationAttemptId().getApplicationId()).
- getTimelineClient();
- putEntityWithoutBlocking(timelineClient, entity);
- } catch (Exception e) {
- LOG.error("Exception in ContainersMonitorImpl in putting " +
- "resource usage metrics to timeline service.", e);
- }
- }
}
if (LOG.isDebugEnabled()) {
LOG.debug("Total Resource Usage stats in NM by all containers : "
@@ -678,20 +596,6 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
}
-
- private void putEntityWithoutBlocking(final TimelineClient timelineClient,
- final TimelineEntity entity) {
- Runnable publishWrapper = new Runnable() {
- public void run() {
- try {
- timelineClient.putEntities(entity);
- } catch (IOException|YarnException e) {
- LOG.error("putEntityNonBlocking get failed: " + e);
- }
- }
- };
- threadPool.execute(publishWrapper);
- }
private String formatErrorMessage(String memTypeExceeded,
long currentVmemUsage, long vmemLimit,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
new file mode 100644
index 0000000..af8d94c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
@@ -0,0 +1,31 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+import org.apache.hadoop.yarn.event.AbstractEvent;
+
+public class NMTimelineEvent extends AbstractEvent<NMTimelineEventType> {
+ public NMTimelineEvent(NMTimelineEventType type) {
+ super(type);
+ }
+
+ public NMTimelineEvent(NMTimelineEventType type, long timestamp) {
+ super(type, timestamp);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
new file mode 100644
index 0000000..c1129af
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
@@ -0,0 +1,24 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+public enum NMTimelineEventType {
+ // Publish the NM Timeline entity
+ TIMELINE_ENTITY_PUBLISH,
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
new file mode 100644
index 0000000..2c5c300
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -0,0 +1,376 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ContainerMetric;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+public class NMTimelinePublisher extends CompositeService {
+
+ private static final Log LOG = LogFactory.getLog(NMTimelinePublisher.class);
+
+ private Dispatcher dispatcher;
+ private boolean publishSystemMetrics;
+
+ private Context context;
+
+ private NodeId nodeId;
+
+ private String httpAddress;
+
+ public NMTimelinePublisher(Context context) {
+ super(NMTimelinePublisher.class.getName());
+ this.context = context;
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ publishSystemMetrics =
+ YarnConfiguration.systemMetricsPublisherEnabled(conf);
+
+ if (publishSystemMetrics) {
+ dispatcher = new AsyncDispatcher();
+ dispatcher.register(NMTimelineEventType.class,
+ new ForwardingEventHandler());
+ dispatcher
+ .register(ContainerEventType.class, new ContainerEventHandler());
+ dispatcher.register(ApplicationEventType.class,
+ new ApplicationEventHandler());
+ dispatcher.register(LocalizationEventType.class,
+ new LocalizationEventDispatcher());
+ addIfService(dispatcher);
+ LOG.info("YARN system metrics publishing service is enabled");
+ } else {
+ LOG.info("YARN system metrics publishing service is not enabled");
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ // context will be updated after containerManagerImpl is started
+ // hence NMMetricsPublisher is added subservice of containerManagerImpl
+ this.nodeId = context.getNodeId();
+ this.httpAddress = nodeId.getHost() + ":" + context.getHttpPort();
+ }
+
+ protected void handleNMTimelineEvent(NMTimelineEvent event) {
+ switch (event.getType()) {
+ case TIMELINE_ENTITY_PUBLISH:
+ putEntity(((TimelinePublishEvent) event).getTimelineEntityToPublish(),
+ ((TimelinePublishEvent) event).getApplicationId());
+ break;
+ default:
+ LOG.error("Unknown NMTimelineEvent type: " + event.getType());
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ public void reportContainerResourceUsage(Container container,
+ long createdTime, String pId, Long pmemUsage,
+ Float cpuUsageTotalCoresPercentage) {
+ if (publishSystemMetrics
+ && (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE || cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE)) {
+ ContainerEntity entity =
+ createContainerEntity(container.getContainerId());
+ long currentTimeMillis = System.currentTimeMillis();
+ if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ TimelineMetric memoryMetric = new TimelineMetric();
+ memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
+ memoryMetric.addValue(currentTimeMillis, pmemUsage);
+ entity.addMetric(memoryMetric);
+ }
+ if (cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ TimelineMetric cpuMetric = new TimelineMetric();
+ cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
+ cpuMetric.addValue(currentTimeMillis, cpuUsageTotalCoresPercentage);
+ entity.addMetric(cpuMetric);
+ }
+ dispatcher.getEventHandler().handle(
+ new TimelinePublishEvent(entity, container.getContainerId()
+ .getApplicationAttemptId().getApplicationId()));
+ }
+ }
+
+ private void publishContainerCreatedEvent(ContainerEntity entity,
+ ContainerId containerId, Resource resource, Priority priority,
+ long timestamp) {
+ Map<String, Object> entityInfo = new HashMap<String, Object>();
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
+ resource.getMemory());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_ENTITY_INFO,
+ resource.getVirtualCores());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_ENTITY_INFO,
+ nodeId.getHost());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
+ nodeId.getPort());
+ entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
+ priority.toString());
+ entityInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ httpAddress);
+ entity.setInfo(entityInfo);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE);
+ tEvent.setTimestamp(timestamp);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
+ }
+
+ private void publishContainerFinishedEvent(ContainerStatus containerStatus,
+ long timeStamp) {
+ ContainerId containerId = containerStatus.getContainerId();
+ TimelineEntity entity = createContainerEntity(containerId);
+
+ Map<String, Object> eventInfo = new HashMap<String, Object>();
+ eventInfo.put(ContainerMetricsConstants.DIAGNOSTICS_INFO_EVENT_INFO,
+ containerStatus.getDiagnostics());
+ eventInfo.put(ContainerMetricsConstants.EXIT_STATUS_EVENT_INFO,
+ containerStatus.getExitStatus());
+ eventInfo.put(ContainerMetricsConstants.STATE_EVENT_INFO, containerStatus
+ .getState().toString());
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(ContainerMetricsConstants.FINISHED_EVENT_TYPE);
+ tEvent.setTimestamp(timeStamp);
+ tEvent.setInfo(eventInfo);
+
+ entity.addEvent(tEvent);
+ putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
+ }
+
+ private static ContainerEntity createContainerEntity(ContainerId containerId) {
+ ContainerEntity entity = new ContainerEntity();
+ entity.setId(containerId.toString());
+ Identifier parentIdentifier = new Identifier();
+ parentIdentifier
+ .setType(TimelineEntityType.YARN_APPLICATION_ATTEMPT.name());
+ parentIdentifier.setId(containerId.getApplicationAttemptId().toString());
+ entity.setParent(parentIdentifier);
+ return entity;
+ }
+
+ private void putEntity(TimelineEntity entity, ApplicationId appId) {
+ try {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Publishing the entity " + entity + ", JSON-style content: "
+ + TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ }
+ TimelineClient timelineClient =
+ context.getApplications().get(appId).getTimelineClient();
+ timelineClient.putEntities(entity);
+ } catch (Exception e) {
+ LOG.error("Error when publishing entity " + entity, e);
+ }
+ }
+
+ public void publishApplicationEvent(ApplicationEvent event) {
+ if (!publishSystemMetrics) {
+ return;
+ }
+ // publish only when the desired event is received
+ switch (event.getType()) {
+ case INIT_APPLICATION:
+ case FINISH_APPLICATION:
+ case APPLICATION_CONTAINER_FINISHED:
+ case APPLICATION_LOG_HANDLING_FAILED:
+ dispatcher.getEventHandler().handle(event);
+ break;
+
+ default:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(event.getType()
+ + " is not a desired ApplicationEvent which needs to be published by"
+ + " NMTimelinePublisher");
+ }
+ break;
+ }
+ }
+
+ public void publishContainerEvent(ContainerEvent event) {
+ if (!publishSystemMetrics) {
+ return;
+ }
+ // publish only when the desired event is received
+ switch (event.getType()) {
+ case INIT_CONTAINER:
+ dispatcher.getEventHandler().handle(event);
+ break;
+
+ default:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(event.getType()
+ + " is not a desired ContainerEvent which needs to be published by"
+ + " NMTimelinePublisher");
+ }
+ break;
+ }
+ }
+
+ public void publishLocalizationEvent(LocalizationEvent event) {
+ if (!publishSystemMetrics) {
+ return;
+ }
+ // publish only when the desired event is received
+ switch (event.getType()) {
+ case CONTAINER_RESOURCES_LOCALIZED:
+ case INIT_CONTAINER_RESOURCES:
+ dispatcher.getEventHandler().handle(event);
+ break;
+
+ default:
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(event.getType()
+ + " is not a desired LocalizationEvent which needs to be published"
+ + " by NMTimelinePublisher");
+ }
+ break;
+ }
+ }
+
+ private class ApplicationEventHandler implements
+ EventHandler<ApplicationEvent> {
+ @Override
+ public void handle(ApplicationEvent event) {
+ switch (event.getType()) {
+ case APPLICATION_CONTAINER_FINISHED:
+ // this is actually used to publish the container Event
+ ApplicationContainerFinishedEvent evnt =
+ (ApplicationContainerFinishedEvent) event;
+ publishContainerFinishedEvent(evnt.getContainerStatus(),
+ event.getTimestamp());
+ break;
+ default:
+ LOG.error("Seems like event type is captured only in "
+ + "publishApplicationEvent method and not handled here");
+ break;
+ }
+ }
+ }
+
+ private class ContainerEventHandler implements EventHandler<ContainerEvent> {
+ @Override
+ public void handle(ContainerEvent event) {
+ ContainerId containerId = event.getContainerID();
+ Container container = context.getContainers().get(containerId);
+ long timestamp = event.getTimestamp();
+ ContainerEntity entity = createContainerEntity(containerId);
+
+ switch (event.getType()) {
+ case INIT_CONTAINER:
+ publishContainerCreatedEvent(entity, containerId,
+ container.getResource(), container.getPriority(), timestamp);
+ break;
+ default:
+ LOG.error("Seems like event type is captured only in "
+ + "publishContainerEvent method and not handled here");
+ break;
+ }
+ }
+ }
+
+ private static final class LocalizationEventDispatcher implements
+ EventHandler<LocalizationEvent> {
+ @Override
+ public void handle(LocalizationEvent event) {
+ switch (event.getType()) {
+ case INIT_CONTAINER_RESOURCES:
+ case CONTAINER_RESOURCES_LOCALIZED:
+ // TODO after priority based flush jira is finished
+ break;
+ default:
+ LOG.error("Seems like event type is captured only in "
+ + "publishLocalizationEvent method and not handled here");
+ break;
+ }
+ }
+ }
+
+ /**
+ * EventHandler implementation which forward events to NMMetricsPublisher.
+ * Making use of it, NMMetricsPublisher can avoid to have a public handle
+ * method.
+ */
+ private final class ForwardingEventHandler implements
+ EventHandler<NMTimelineEvent> {
+
+ @Override
+ public void handle(NMTimelineEvent event) {
+ handleNMTimelineEvent(event);
+ }
+ }
+
+ private static class TimelinePublishEvent extends NMTimelineEvent {
+ private ApplicationId appId;
+ private TimelineEntity entityToPublish;
+
+ public TimelinePublishEvent(TimelineEntity entity, ApplicationId appId) {
+ super(NMTimelineEventType.TIMELINE_ENTITY_PUBLISH, System
+ .currentTimeMillis());
+ this.appId = appId;
+ this.entityToPublish = entity;
+ }
+
+ public ApplicationId getApplicationId() {
+ return appId;
+ }
+
+ public TimelineEntity getTimelineEntityToPublish() {
+ return entityToPublish;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index 7d895f2..07eea98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.proto.YarnServerCommonServiceProtos.NodeHeartbeatResponseProto;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.ContainerContext;
import org.apache.hadoop.yarn.server.api.ResourceTracker;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
@@ -244,8 +245,10 @@ public class TestNodeStatusUpdater {
firstContainerID, InetAddress.getByName("localhost")
.getCanonicalHostName(), 1234, user, resource,
currentTime + 10000, 123, "password".getBytes(), currentTime));
- Container container = new ContainerImpl(conf, mockDispatcher,
- stateStore, launchContext, null, mockMetrics, containerToken);
+ Container container =
+ new ContainerImpl(conf, mockDispatcher, stateStore, launchContext,
+ null, mockMetrics, containerToken,
+ mock(Context.class));
this.context.getContainers().put(firstContainerID, container);
} else if (heartBeatID == 2) {
// Checks on the RM end
@@ -274,8 +277,10 @@ public class TestNodeStatusUpdater {
secondContainerID, InetAddress.getByName("localhost")
.getCanonicalHostName(), 1234, user, resource,
currentTime + 10000, 123, "password".getBytes(), currentTime));
- Container container = new ContainerImpl(conf, mockDispatcher,
- stateStore, launchContext, null, mockMetrics, containerToken);
+ Container container =
+ new ContainerImpl(conf, mockDispatcher, stateStore, launchContext,
+ null, mockMetrics, containerToken,
+ mock(Context.class));
this.context.getContainers().put(secondContainerID, container);
} else if (heartBeatID == 3) {
// Checks on the RM end
@@ -934,7 +939,8 @@ public class TestNodeStatusUpdater {
"password".getBytes(), 0);
Container anyCompletedContainer = new ContainerImpl(conf, null,
null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),
+ mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
@@ -955,7 +961,7 @@ public class TestNodeStatusUpdater {
"password".getBytes(), 0);
Container runningContainer =
new ContainerImpl(conf, null, null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(runningContainerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(runningContainerToken), mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
return ContainerState.RUNNING;
@@ -1011,9 +1017,10 @@ public class TestNodeStatusUpdater {
BuilderUtils.newContainerToken(containerId, "host", 1234, "user",
BuilderUtils.newResource(1024, 1), 0, 123,
"password".getBytes(), 0);
+
Container completedContainer = new ContainerImpl(conf, null,
null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
return ContainerState.COMPLETE;
@@ -1050,7 +1057,8 @@ public class TestNodeStatusUpdater {
"password".getBytes(), 0);
Container anyCompletedContainer = new ContainerImpl(conf, null,
null, null, null, null,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),
+ mock(Context.class)) {
@Override
public ContainerState getCurrentState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
index 1380752..8c4bd25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
@@ -25,6 +25,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
import java.io.File;
import java.io.IOException;
@@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.junit.Assert;
@@ -193,7 +195,7 @@ public class TestAuxServices {
ContainerId.newContainerId(attemptId, 1), "", "",
Resource.newInstance(1, 1), 0,0,0, Priority.newInstance(0), 0);
Container container = new ContainerImpl(null, null, null, null, null,
- null, cti);
+ null, cti, mock(Context.class));
ContainerId containerId = container.getContainerId();
Resource resource = container.getResource();
event = new AuxServicesEvent(AuxServicesEventType.CONTAINER_INIT,container);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 9c3109d..c8f3008 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -76,8 +76,10 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreServic
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestContainerManagerRecovery {
@@ -419,6 +421,12 @@ public class TestContainerManagerRecovery {
boolean blockNewContainerRequests) {
// do nothing
}
+
+ @Override
+ public NMTimelinePublisher createNMTimelinePublisher(Context context) {
+ NMTimelinePublisher timelinePublisher = mock(NMTimelinePublisher.class);
+ return timelinePublisher;
+ }
};
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 002d4cf..28b5538 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -579,7 +581,7 @@ public class TestApplication {
public void containerFinished(int containerNum) {
app.handle(new ApplicationContainerFinishedEvent(containers.get(
- containerNum).getContainerId()));
+ containerNum).cloneAndGetContainerStatus()));
drainDispatcherEvents();
}
@@ -616,6 +618,8 @@ public class TestApplication {
when(c.getLaunchContext()).thenReturn(launchContext);
when(launchContext.getApplicationACLs()).thenReturn(
new HashMap<ApplicationAccessType, String>());
+ when(c.cloneAndGetContainerStatus()).thenReturn(BuilderUtils.newContainerStatus(cId,
+ ContainerState.NEW, "", 0));
return c;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 2834e30..aa22e5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -808,7 +808,7 @@ public class TestContainer {
when(ctxt.getServiceData()).thenReturn(serviceData);
c = new ContainerImpl(conf, dispatcher, new NMNullStateStoreService(),
- ctxt, null, metrics, identifier);
+ ctxt, null, metrics, identifier, mock(Context.class));
dispatcher.register(ContainerEventType.class,
new EventHandler<ContainerEvent>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index b2ccb61..c5bdbb4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -140,4 +141,9 @@ public class MockContainer implements Container {
public NMContainerStatus getNMContainerStatus() {
return null;
}
+
+ @Override
+ public Priority getPriority() {
+ return Priority.UNDEFINED;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
index ed94fb6..e839f47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
@@ -220,7 +220,8 @@ public class TestNMWebServer {
Container container =
new ContainerImpl(conf, dispatcher, stateStore, launchContext,
null, metrics,
- BuilderUtils.newContainerTokenIdentifier(containerToken)) {
+ BuilderUtils.newContainerTokenIdentifier(containerToken),
+ mock(Context.class)) {
@Override
public ContainerState getContainerState() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index befaa83..4147d42 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -137,9 +137,19 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
// intercept the event of the AM container being stopped and remove the app
// level collector service
if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
- ApplicationId appId = context.getContainerId().
- getApplicationAttemptId().getApplicationId();
- removeApplication(appId);
+ final ApplicationId appId =
+ context.getContainerId().getApplicationAttemptId().getApplicationId();
+ new Thread(new Runnable() {
+ public void run() {
+ try {
+ // TODO Temporary Fix until solution for YARN-3995 is finalized.
+ Thread.sleep(1000l);
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ removeApplication(appId);
+ }
+ }).start();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/702a2142/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index 7cc612d..dafc76e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -98,6 +98,15 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerType()).thenReturn(
ContainerType.APPLICATION_MASTER);
auxService.stopContainer(context);
+
+ // TODO Temporary Fix until solution for YARN-3995 is finalized
+ for (int i = 0; i < 4; i++) {
+ Thread.sleep(500l);
+ if (!auxService.hasApplication(appAttemptId.getApplicationId())) {
+ break;
+ }
+ }
+
// auxService should not have that app
assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
auxService.close();
[03/50] [abbrv] hadoop git commit: YARN-3034. Implement RM starting
its timeline collector. Contributed by Naganarasimha G R
Posted by sj...@apache.org.
YARN-3034. Implement RM starting its timeline collector. Contributed by Naganarasimha G R
(cherry picked from commit dc12cad2b89f643dafa0def863325cb374c7670c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a7985239
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a7985239
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a7985239
Branch: refs/heads/YARN-2928
Commit: a798523971852d15a4a9098f4e689e2c801fab06
Parents: afa9925
Author: Junping Du <ju...@apache.org>
Authored: Tue Mar 24 13:42:14 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:38:43 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 14 ++-
.../src/main/resources/yarn-default.xml | 11 +-
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 +
.../resourcemanager/RMActiveServiceContext.java | 43 ++++++++
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 14 ++-
.../server/resourcemanager/ResourceManager.java | 44 ++++++--
.../metrics/SystemMetricsPublisher.java | 29 +++---
.../timelineservice/RMTimelineCollector.java | 104 +++++++++++++++++++
10 files changed, 241 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d9696c9..82ed275 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -38,6 +38,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration.
(Sangjin Lee via zjshen)
+ YARN-3034. Implement RM starting its timeline collector. (Naganarasimha G R
+ via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 9d5b63b..e72c1c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -373,12 +373,20 @@ public class YarnConfiguration extends Configuration {
/**
* The setting that controls whether yarn system metrics is published on the
- * timeline server or not by RM.
+ * timeline server or not by RM. This configuration setting is for ATS V1
*/
- public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED =
- RM_PREFIX + "system-metrics-publisher.enabled";
+ public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = RM_PREFIX
+ + "system-metrics-publisher.enabled";
public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
+ /**
+ * The setting that controls whether yarn system metrics is published on the
+ * timeline server or not by RM and NM. This configuration setting is for ATS V2
+ */
+ public static final String SYSTEM_METRICS_PUBLISHER_ENABLED = YARN_PREFIX
+ + "system-metrics-publisher.enabled";
+ public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
+
public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size";
public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 78b6ae8..10450a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -746,12 +746,21 @@
<property>
<description>The setting that controls whether yarn system metrics is
- published on the timeline server or not by RM.</description>
+ published to the Timeline server (version one) or not, by RM.
+ This configuration is deprecated.</description>
<name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
<property>
+ <description>The setting that controls whether yarn system metrics is
+ published on the Timeline server (version two) or not by RM And NM.</description>
+ <name>yarn.system-metrics-publisher.enabled</name>
+ <value>false</value>
+ </property>
+
+
+ <property>
<description>Number of worker threads that send the yarn system metrics
data.</description>
<name>yarn.resourcemanager.system-metrics-publisher.dispatcher.pool-size</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 9d54184..b88815f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -171,6 +171,10 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-server-timelineservice</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
</dependency>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 1abb14e..d79e542 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
@@ -92,6 +93,10 @@ public class RMActiveServiceContext {
private NodesListManager nodesListManager;
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
+ private RMApplicationHistoryWriter rmApplicationHistoryWriter;
+ private SystemMetricsPublisher systemMetricsPublisher;
+ private RMTimelineCollector timelineCollector;
+
private RMNodeLabelsManager nodeLabelManager;
private long epoch;
private Clock systemClock = new SystemClock();
@@ -366,6 +371,44 @@ public class RMActiveServiceContext {
@Private
@Unstable
+ public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
+ return rmApplicationHistoryWriter;
+ }
+
+ @Private
+ @Unstable
+ public RMTimelineCollector getRMTimelineCollector() {
+ return timelineCollector;
+ }
+
+ @Private
+ @Unstable
+ public void setRMTimelineCollector(RMTimelineCollector timelineCollector) {
+ this.timelineCollector = timelineCollector;
+ }
+
+ @Private
+ @Unstable
+ public void setSystemMetricsPublisher(
+ SystemMetricsPublisher systemMetricsPublisher) {
+ this.systemMetricsPublisher = systemMetricsPublisher;
+ }
+
+ @Private
+ @Unstable
+ public SystemMetricsPublisher getSystemMetricsPublisher() {
+ return systemMetricsPublisher;
+ }
+
+ @Private
+ @Unstable
+ public void setRMApplicationHistoryWriter(
+ RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
+ }
+
+ @Private
+ @Unstable
public long getEpoch() {
return this.epoch;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index bc50268..05fee99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
/**
* Context of the ResourceManager.
@@ -108,6 +109,10 @@ public interface RMContext {
void setSystemMetricsPublisher(SystemMetricsPublisher systemMetricsPublisher);
SystemMetricsPublisher getSystemMetricsPublisher();
+
+ void setRMTimelineCollector(RMTimelineCollector timelineCollector);
+
+ RMTimelineCollector getRMTimelineCollector();
ConfigurationProvider getConfigurationProvider();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index d6d573d..8543cec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -22,8 +22,8 @@ import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.LocalConfigurationProvider;
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
@@ -354,6 +355,17 @@ public class RMContextImpl implements RMContext {
}
@Override
+ public void setRMTimelineCollector(
+ RMTimelineCollector timelineCollector) {
+ activeServiceContext.setRMTimelineCollector(timelineCollector);
+ }
+
+ @Override
+ public RMTimelineCollector getRMTimelineCollector() {
+ return activeServiceContext.getRMTimelineCollector();
+ }
+
+ @Override
public void setSystemMetricsPublisher(
SystemMetricsPublisher systemMetricsPublisher) {
this.systemMetricsPublisher = systemMetricsPublisher;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index d6d9629..e5c72cd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -18,7 +18,16 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.PrintStream;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -80,11 +89,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeEventType;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.*;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter;
@@ -99,15 +111,7 @@ import org.apache.hadoop.yarn.webapp.WebApps;
import org.apache.hadoop.yarn.webapp.WebApps.Builder;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.PrintStream;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.LinkedBlockingQueue;
+import com.google.common.annotations.VisibleForTesting;
/**
* The ResourceManager is the main class that is a set of components.
@@ -364,6 +368,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
return new RMApplicationHistoryWriter();
}
+ private RMTimelineCollector createRMTimelineCollector() {
+ return new RMTimelineCollector();
+ }
+
protected SystemMetricsPublisher createSystemMetricsPublisher() {
return new SystemMetricsPublisher();
}
@@ -476,6 +484,20 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setDelegationTokenRenewer(delegationTokenRenewer);
}
+ RMApplicationHistoryWriter rmApplicationHistoryWriter =
+ createRMApplicationHistoryWriter();
+ addService(rmApplicationHistoryWriter);
+ rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+
+ SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher();
+ addService(systemMetricsPublisher);
+ rmContext.setSystemMetricsPublisher(systemMetricsPublisher);
+
+ RMTimelineCollector timelineCollector =
+ createRMTimelineCollector();
+ addService(timelineCollector);
+ rmContext.setRMTimelineCollector(timelineCollector);
+
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 3d7ac9f..b4ce4f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -53,7 +53,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
/**
- * The class that helps RM publish metrics to the timeline server. RM will
+ * The class that helps RM publish metrics to the timeline server V1. RM will
* always invoke the methods of this class regardless the service is enabled or
* not. If it is disabled, publishing requests will be ignored silently.
*/
@@ -66,7 +66,7 @@ public class SystemMetricsPublisher extends CompositeService {
private Dispatcher dispatcher;
private TimelineClient client;
- private boolean publishSystemMetrics;
+ private boolean publishSystemMetricsToATSv1;
public SystemMetricsPublisher() {
super(SystemMetricsPublisher.class.getName());
@@ -74,13 +74,14 @@ public class SystemMetricsPublisher extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
- publishSystemMetrics =
+ publishSystemMetricsToATSv1 =
conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) &&
- conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
- YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED);
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
+ && conf.getBoolean(
+ YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
client = TimelineClient.createTimelineClient();
addIfService(client);
@@ -97,7 +98,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appCreated(RMApp app, long createdTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new ApplicationCreatedEvent(
app.getApplicationId(),
@@ -114,7 +115,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appFinished(RMApp app, RMAppState state, long finishedTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new ApplicationFinishedEvent(
app.getApplicationId(),
@@ -131,7 +132,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appACLsUpdated(RMApp app, String appViewACLs,
long updatedTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new ApplicationACLsUpdatedEvent(
app.getApplicationId(),
@@ -143,7 +144,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appAttemptRegistered(RMAppAttempt appAttempt,
long registeredTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new AppAttemptRegisteredEvent(
appAttempt.getAppAttemptId(),
@@ -159,7 +160,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void appAttemptFinished(RMAppAttempt appAttempt,
RMAppAttemptState appAttemtpState, RMApp app, long finishedTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new AppAttemptFinishedEvent(
appAttempt.getAppAttemptId(),
@@ -176,7 +177,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void containerCreated(RMContainer container, long createdTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new ContainerCreatedEvent(
container.getContainerId(),
@@ -189,7 +190,7 @@ public class SystemMetricsPublisher extends CompositeService {
@SuppressWarnings("unchecked")
public void containerFinished(RMContainer container, long finishedTime) {
- if (publishSystemMetrics) {
+ if (publishSystemMetricsToATSv1) {
dispatcher.getEventHandler().handle(
new ContainerFinishedEvent(
container.getContainerId(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a7985239/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
new file mode 100644
index 0000000..22743d6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.timelineservice;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEventType;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
+
+/**
+ * This class is responsible for posting application and appattempt lifecycle
+ * related events to timeline service V2
+ */
+@Private
+@Unstable
+public class RMTimelineCollector extends TimelineCollector {
+ private static final Log LOG = LogFactory.getLog(RMTimelineCollector.class);
+
+ public RMTimelineCollector() {
+ super("Resource Manager TimelineCollector");
+ }
+
+ private Dispatcher dispatcher;
+
+ private boolean publishSystemMetricsForV2;
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ publishSystemMetricsForV2 =
+ conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
+ && conf.getBoolean(
+ YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
+
+ if (publishSystemMetricsForV2) {
+ // having separate dispatcher to avoid load on RMDispatcher
+ LOG.info("RMTimelineCollector has been configured to publish"
+ + " System Metrics in ATS V2");
+ dispatcher = new AsyncDispatcher();
+ dispatcher.register(SystemMetricsEventType.class,
+ new ForwardingEventHandler());
+ } else {
+ LOG.warn("RMTimelineCollector has not been configured to publish"
+ + " System Metrics in ATS V2");
+ }
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ protected void handleSystemMetricsEvent(SystemMetricsEvent event) {
+ switch (event.getType()) {
+ default:
+ LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
+ }
+ }
+
+ /**
+ * EventHandler implementation which forward events to SystemMetricsPublisher.
+ * Making use of it, SystemMetricsPublisher can avoid to have a public handle
+ * method.
+ */
+ private final class ForwardingEventHandler implements
+ EventHandler<SystemMetricsEvent> {
+
+ @Override
+ public void handle(SystemMetricsEvent event) {
+ handleSystemMetricsEvent(event);
+ }
+ }
+}
[14/50] [abbrv] hadoop git commit: MAPREDUCE-6335. Created MR job
based performance test driver for the timeline service v2. Contributed by
Sangjin Lee.
Posted by sj...@apache.org.
MAPREDUCE-6335. Created MR job based performance test driver for the timeline service v2. Contributed by Sangjin Lee.
(cherry picked from commit b689f5d43d3f5434a30fe52f1a7e12e1fc5c71f4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8f0b1cae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8f0b1cae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8f0b1cae
Branch: refs/heads/YARN-2928
Commit: 8f0b1cae5e1fd467efa6cc773ff744b27f05a2b3
Parents: 8e58f94
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue Apr 28 19:46:01 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:10 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../mapred/TimelineServicePerformanceV2.java | 298 +++++++++++++++++++
.../apache/hadoop/test/MapredTestDriver.java | 3 +
3 files changed, 304 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f0b1cae/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 5ac0d3b..2805780 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -9,6 +9,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
MAPREDUCE-6327. Made MR AM use timeline service v2 API to write history
events and counters. (Junping Du via zjshen)
+ MAPREDUCE-6335. Created MR job based performance test driver for the
+ timeline service v2. (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f0b1cae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
new file mode 100644
index 0000000..de46617
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -0,0 +1,298 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Date;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.SleepJob.SleepInputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+
+public class TimelineServicePerformanceV2 extends Configured implements Tool {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineServicePerformanceV2.class);
+
+ static final int NUM_MAPS_DEFAULT = 1;
+
+ static final int SIMPLE_ENTITY_WRITER = 1;
+ // constants for mtype = 1
+ static final String KBS_SENT = "kbs sent";
+ static final int KBS_SENT_DEFAULT = 1;
+ static final String TEST_TIMES = "testtimes";
+ static final int TEST_TIMES_DEFAULT = 100;
+ static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
+ "timeline.server.performance.run.id";
+
+ static int mapperType = SIMPLE_ENTITY_WRITER;
+
+ protected static int printUsage() {
+ // TODO is there a way to handle mapper-specific options more gracefully?
+ System.err.println(
+ "Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
+ ")\n" +
+ " [-mtype <mapper type in integer>] \n" +
+ " 1. simple entity write mapper\n" +
+ " [-s <(KBs)test>] number of KB per put (default: " +
+ KBS_SENT_DEFAULT + " KB)\n" +
+ " [-t] package sending iterations per mapper (default: " +
+ TEST_TIMES_DEFAULT + ")\n");
+ GenericOptionsParser.printGenericCommandUsage(System.err);
+ return -1;
+ }
+
+ /**
+ * Configure a job given argv.
+ */
+ public static boolean parseArgs(String[] args, Job job) throws IOException {
+ // set the defaults
+ Configuration conf = job.getConfiguration();
+ conf.setInt(MRJobConfig.NUM_MAPS, NUM_MAPS_DEFAULT);
+ conf.setInt(KBS_SENT, KBS_SENT_DEFAULT);
+ conf.setInt(TEST_TIMES, TEST_TIMES_DEFAULT);
+
+ for (int i = 0; i < args.length; i++) {
+ if (args.length == i + 1) {
+ System.out.println("ERROR: Required parameter missing from " + args[i]);
+ return printUsage() == 0;
+ }
+ try {
+ if ("-m".equals(args[i])) {
+ if (Integer.parseInt(args[++i]) > 0) {
+ job.getConfiguration()
+ .setInt(MRJobConfig.NUM_MAPS, (Integer.parseInt(args[i])));
+ }
+ } else if ("-mtype".equals(args[i])) {
+ mapperType = Integer.parseInt(args[++i]);
+ switch (mapperType) {
+ case SIMPLE_ENTITY_WRITER:
+ job.setMapperClass(SimpleEntityWriter.class);
+ break;
+ default:
+ job.setMapperClass(SimpleEntityWriter.class);
+ }
+ } else if ("-s".equals(args[i])) {
+ if (Integer.parseInt(args[++i]) > 0) {
+ conf.setInt(KBS_SENT, (Integer.parseInt(args[i])));
+ }
+ } else if ("-t".equals(args[i])) {
+ if (Integer.parseInt(args[++i]) > 0) {
+ conf.setInt(TEST_TIMES, (Integer.parseInt(args[i])));
+ }
+ } else {
+ System.out.println("Unexpected argument: " + args[i]);
+ return printUsage() == 0;
+ }
+ } catch (NumberFormatException except) {
+ System.out.println("ERROR: Integer expected instead of " + args[i]);
+ return printUsage() == 0;
+ } catch (Exception e) {
+ throw (IOException)new IOException().initCause(e);
+ }
+ }
+
+ return true;
+ }
+
+ /**
+ * TimelineServer Performance counters
+ */
+ static enum PerfCounters {
+ TIMELINE_SERVICE_WRITE_TIME,
+ TIMELINE_SERVICE_WRITE_COUNTER,
+ TIMELINE_SERVICE_WRITE_FAILURES,
+ TIMELINE_SERVICE_WRITE_KBS,
+ }
+
+ public int run(String[] args) throws Exception {
+
+ Job job = Job.getInstance(getConf());
+ job.setJarByClass(TimelineServicePerformanceV2.class);
+ job.setMapperClass(SimpleEntityWriter.class);
+ job.setInputFormatClass(SleepInputFormat.class);
+ job.setOutputFormatClass(NullOutputFormat.class);
+ job.setNumReduceTasks(0);
+ if (!parseArgs(args, job)) {
+ return -1;
+ }
+
+ // for mtype = 1
+ // use the current timestamp as the "run id" of the test: this will be used
+ // as simulating the cluster timestamp for apps
+ Configuration conf = job.getConfiguration();
+ conf.setLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
+ System.currentTimeMillis());
+
+ Date startTime = new Date();
+ System.out.println("Job started: " + startTime);
+ int ret = job.waitForCompletion(true) ? 0 : 1;
+ org.apache.hadoop.mapreduce.Counters counters = job.getCounters();
+ long writetime =
+ counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).getValue();
+ long writecounts =
+ counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).getValue();
+ long writesize =
+ counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
+ double transacrate = writecounts * 1000 / (double)writetime;
+ double iorate = writesize * 1000 / (double)writetime;
+ int numMaps = Integer.parseInt(conf.get(MRJobConfig.NUM_MAPS));
+
+ System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
+ " ops/s");
+ System.out.println("IO RATE (per mapper): " + iorate + " KB/s");
+
+ System.out.println("TRANSACTION RATE (total): " + transacrate*numMaps +
+ " ops/s");
+ System.out.println("IO RATE (total): " + iorate*numMaps + " KB/s");
+
+ return ret;
+ }
+
+ public static void main(String[] args) throws Exception {
+ int res =
+ ToolRunner.run(new Configuration(), new TimelineServicePerformanceV2(),
+ args);
+ System.exit(res);
+ }
+
+ /**
+ * To ensure that the compression really gets exercised, generate a
+ * random alphanumeric fixed length payload
+ */
+ static final char[] alphaNums = new char[] { 'a', 'b', 'c', 'd', 'e', 'f',
+ 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
+ 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
+ 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+ 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
+ '3', '4', '5', '6', '7', '8', '9', '0', ' ' };
+
+ /**
+ * Adds simple entities with random string payload, events, metrics, and
+ * configuration.
+ */
+ public static class SimpleEntityWriter
+ extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
+ public void map(IntWritable key, IntWritable val, Context context)
+ throws IOException {
+
+ Configuration conf = context.getConfiguration();
+ // simulate the app id with the task id
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
+ ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
+
+ // create the app level timeline collector
+ Configuration tlConf = new YarnConfiguration();
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ collector.init(tlConf);
+ collector.start();
+
+ try {
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(context.getJobName());
+ tlContext.setFlowRunId(timestamp);
+ tlContext.setUserId(context.getUser());
+
+ final int kbs = Integer.parseInt(conf.get(KBS_SENT));
+
+ long totalTime = 0;
+ final int testtimes = Integer.parseInt(conf.get(TEST_TIMES));
+ final Random rand = new Random();
+ final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
+ final char[] payLoad = new char[kbs * 1024];
+
+ for (int i = 0; i < testtimes; i++) {
+ // Generate a fixed length random payload
+ for (int xx = 0; xx < kbs * 1024; xx++) {
+ int alphaNumIdx = rand.nextInt(alphaNums.length);
+ payLoad[xx] = alphaNums[alphaNumIdx];
+ }
+ String entId = taskAttemptId + "_" + Integer.toString(i);
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId(entId);
+ entity.setType("FOO_ATTEMPT");
+ entity.addInfo("PERF_TEST", payLoad);
+ // add an event
+ TimelineEvent event = new TimelineEvent();
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+ entity.addEvent(event);
+ // add a metric
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("foo_metric");
+ metric.setSingleData(123456789L);
+ entity.addMetric(metric);
+ // add a config
+ entity.addConfig("foo", "bar");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ collector.putEntities(entities, ugi);
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ e.printStackTrace();
+ }
+ long endWrite = System.nanoTime();
+ totalTime += (endWrite-startWrite)/1000000L;
+ }
+ LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
+ " kB) in " + totalTime + " ms");
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(testtimes);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
+ increment(kbs*testtimes);
+ } finally {
+ // clean up
+ collector.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8f0b1cae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
index 8fa82aa..5dcb143 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.mapred.TestMapRed;
import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
import org.apache.hadoop.mapred.TestTextInputFormat;
import org.apache.hadoop.mapred.ThreadedMapBenchmark;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2;
import org.apache.hadoop.mapreduce.FailJob;
import org.apache.hadoop.mapreduce.LargeSorter;
import org.apache.hadoop.mapreduce.MiniHadoopClusterManager;
@@ -90,6 +91,8 @@ public class MapredTestDriver {
pgd.addClass("fail", FailJob.class, "a job that always fails");
pgd.addClass("sleep", SleepJob.class,
"A job that sleeps at each map and reduce task.");
+ pgd.addClass("timelineperformance", TimelineServicePerformanceV2.class,
+ "A job that launch mappers to test timline service performance.");
pgd.addClass("nnbench", NNBench.class,
"A benchmark that stresses the namenode w/ MR.");
pgd.addClass("nnbenchWithoutMR", NNBenchWithoutMR.class,
[15/50] [abbrv] hadoop git commit: YARN-3551. Consolidate data model
change according to the backend implementation (Zhijie Shen via sale)
Posted by sj...@apache.org.
YARN-3551. Consolidate data model change according to the backend implementation (Zhijie Shen via sale)
(cherry picked from commit 557a3950bddc837469244835f5577899080115d8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84147698
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84147698
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84147698
Branch: refs/heads/YARN-2928
Commit: 841476988c54bdb42a9837951e7cc36a2f6bb38a
Parents: 8f0b1ca
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon May 4 16:10:20 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:10 2015 -0700
----------------------------------------------------------------------
.../mapred/TimelineServicePerformanceV2.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 16 +--
.../records/timelineservice/TimelineMetric.java | 131 +++++++++----------
.../TestTimelineServiceRecords.java | 81 +++++++++---
.../monitor/ContainersMonitorImpl.java | 4 +-
.../TestTimelineServiceClientIntegration.java | 6 +
7 files changed, 146 insertions(+), 97 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
index de46617..1c2e28d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -261,7 +261,7 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
// add a metric
TimelineMetric metric = new TimelineMetric();
metric.setId("foo_metric");
- metric.setSingleData(123456789L);
+ metric.addValue(System.currentTimeMillis(), 123456789L);
entity.addMetric(metric);
// add a config
entity.addConfig("foo", "bar");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 63243fb..41ed1ec 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -58,6 +58,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3431. Sub resources of timeline entity needs to be passed to a separate
endpoint. (Zhijie Shen via junping_du)
+ YARN-3551. Consolidate data model change according to the backend
+ implementation (Zhijie Shen via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 6cab753..3be7f52 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -80,7 +80,7 @@ public class TimelineEntity {
private TimelineEntity real;
private Identifier identifier;
private HashMap<String, Object> info = new HashMap<>();
- private HashMap<String, Object> configs = new HashMap<>();
+ private HashMap<String, String> configs = new HashMap<>();
private Set<TimelineMetric> metrics = new HashSet<>();
private Set<TimelineEvent> events = new HashSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
@@ -213,7 +213,7 @@ public class TimelineEntity {
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "configs")
- public HashMap<String, Object> getConfigsJAXB() {
+ public HashMap<String, String> getConfigsJAXB() {
if (real == null) {
return configs;
} else {
@@ -221,7 +221,7 @@ public class TimelineEntity {
}
}
- public Map<String, Object> getConfigs() {
+ public Map<String, String> getConfigs() {
if (real == null) {
return configs;
} else {
@@ -229,19 +229,19 @@ public class TimelineEntity {
}
}
- public void setConfigs(Map<String, Object> configs) {
+ public void setConfigs(Map<String, String> configs) {
if (real == null) {
if (configs != null && !(configs instanceof HashMap)) {
- this.configs = new HashMap<String, Object>(configs);
+ this.configs = new HashMap<String, String>(configs);
} else {
- this.configs = (HashMap<String, Object>) configs;
+ this.configs = (HashMap<String, String>) configs;
}
} else {
real.setConfigs(configs);
}
}
- public void addConfigs(Map<String, Object> configs) {
+ public void addConfigs(Map<String, String> configs) {
if (real == null) {
this.configs.putAll(configs);
} else {
@@ -249,7 +249,7 @@ public class TimelineEntity {
}
}
- public void addConfig(String key, Object value) {
+ public void addConfig(String key, String value) {
if (real == null) {
configs.put(key, value);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 57babbf..c897d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -24,113 +24,104 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
+import java.util.Comparator;
import java.util.Map;
+import java.util.TreeMap;
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineMetric {
- private String id;
- private HashMap<String, Object> info = new HashMap<>();
- private Object singleData;
- private HashMap<Long, Object> timeSeries = new LinkedHashMap<>();
- private long startTime;
- private long endTime;
-
- public TimelineMetric() {
+ public static enum Type {
+ SINGLE_VALUE,
+ TIME_SERIES
}
- @XmlElement(name = "id")
- public String getId() {
- return id;
- }
-
- public void setId(String id) {
- this.id = id;
- }
+ private Type type;
+ private String id;
+ private Comparator<Long> reverseComparator = new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return -l1.compareTo(l2);
+ }
+ };
+ private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
- // required by JAXB
- @InterfaceAudience.Private
- @XmlElement(name = "info")
- public HashMap<String, Object> getInfoJAXB() {
- return info;
+ public TimelineMetric() {
+ this(Type.SINGLE_VALUE);
}
- public Map<String, Object> getInfo() {
- return info;
+ public TimelineMetric(Type type) {
+ this.type = type;
}
- public void setInfo(Map<String, Object> info) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
- }
- public void addInfo(Map<String, Object> info) {
- this.info.putAll(info);
+ @XmlElement(name = "type")
+ public Type getType() {
+ return type;
}
- public void addInfo(String key, Object value) {
- info.put(key, value);
+ public void setType(Type type) {
+ this.type = type;
}
- @XmlElement(name = "data")
- public Object getSingleData() {
- return singleData;
+ @XmlElement(name = "id")
+ public String getId() {
+ return id;
}
- public void setSingleData(Object singleData) {
- this.singleData = singleData;
+ public void setId(String id) {
+ this.id = id;
}
// required by JAXB
@InterfaceAudience.Private
- @XmlElement(name = "timeseries")
- public HashMap<Long, Object> getTimeSeriesJAXB() {
- return timeSeries;
+ @XmlElement(name = "values")
+ public TreeMap<Long, Number> getValuesJAXB() {
+ return values;
}
- public Map<Long, Object> getTimeSeries() {
- return timeSeries;
+ public Map<Long, Number> getValues() {
+ return values;
}
- public void setTimeSeries(Map<Long, Object> timeSeries) {
- if (timeSeries != null && !(timeSeries instanceof LinkedHashMap)) {
- this.timeSeries = new LinkedHashMap<Long, Object>(timeSeries);
+ public void setValues(Map<Long, Number> values) {
+ if (type == Type.SINGLE_VALUE) {
+ overwrite(values);
} else {
- this.timeSeries = (LinkedHashMap<Long, Object>) timeSeries;
+ if (values != null) {
+ this.values = new TreeMap<Long, Number>(reverseComparator);
+ this.values.putAll(values);
+ } else {
+ this.values = null;
+ }
}
}
- public void addTimeSeries(Map<Long, Object> timeSeries) {
- this.timeSeries.putAll(timeSeries);
- }
-
- public void addTimeSeriesData(long timestamp, Object value) {
- timeSeries.put(timestamp, value);
- }
-
- @XmlElement(name = "starttime")
- public long getStartTime() {
- return startTime;
- }
-
- public void setStartTime(long startTime) {
- this.startTime = startTime;
+ public void addValues(Map<Long, Number> values) {
+ if (type == Type.SINGLE_VALUE) {
+ overwrite(values);
+ } else {
+ this.values.putAll(values);
+ }
}
- @XmlElement(name = "endtime")
- public long getEndTime() {
- return endTime;
+ public void addValue(long timestamp, Number value) {
+ if (type == Type.SINGLE_VALUE) {
+ values.clear();
+ }
+ values.put(timestamp, value);
}
- public void setEndTime(long endTime) {
- this.endTime = endTime;
+ private void overwrite(Map<Long, Number> values) {
+ if (values.size() > 1) {
+ throw new IllegalArgumentException(
+ "Values cannot contain more than one point in " +
+ Type.SINGLE_VALUE + " mode");
+ }
+ this.values.clear();
+ this.values.putAll(values);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index caa3f3f..95b922e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -28,7 +28,11 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
import org.junit.Assert;
+import java.util.Arrays;
import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
public class TestTimelineServiceRecords {
@@ -41,38 +45,83 @@ public class TestTimelineServiceRecords {
entity.setType("test type 1");
entity.setId("test id 1");
entity.addInfo("test info key 1", "test info value 1");
- entity.addInfo("test info key 2", "test info value 2");
+ entity.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ entity.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ entity.getInfo().get("test info key 3") instanceof Boolean);
entity.addConfig("test config key 1", "test config value 1");
entity.addConfig("test config key 2", "test config value 2");
- TimelineMetric metric1 = new TimelineMetric();
+
+ TimelineMetric metric1 =
+ new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
metric1.setId("test metric id 1");
- metric1.addInfo("test info key 1", "test info value 1");
- metric1.addInfo("test info key 2", "test info value 2");
- metric1.addTimeSeriesData(1L, "test time series 1");
- metric1.addTimeSeriesData(2L, "test time series 2");
- metric1.setStartTime(0L);
- metric1.setEndTime(1L);
+ metric1.addValue(1L, 1.0F);
+ metric1.addValue(3L, 3.0D);
+ metric1.addValue(2L, 2);
+ Assert.assertEquals(TimelineMetric.Type.TIME_SERIES, metric1.getType());
+ Iterator<Map.Entry<Long, Number>> itr =
+ metric1.getValues().entrySet().iterator();
+ Map.Entry<Long, Number> entry = itr.next();
+ Assert.assertEquals(new Long(3L), entry.getKey());
+ Assert.assertEquals(new Double(3.0D), entry.getValue());
+ entry = itr.next();
+ Assert.assertEquals(new Long(2L), entry.getKey());
+ Assert.assertEquals(new Integer(2), entry.getValue());
+ entry = itr.next();
+ Assert.assertEquals(new Long(1L), entry.getKey());
+ Assert.assertEquals(new Float(1.0F), entry.getValue());
+ Assert.assertFalse(itr.hasNext());
entity.addMetric(metric1);
- TimelineMetric metric2 = new TimelineMetric();
+
+ TimelineMetric metric2 =
+ new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
metric2.setId("test metric id 1");
- metric2.addInfo("test info key 1", "test info value 1");
- metric2.addInfo("test info key 2", "test info value 2");
- metric2.setSingleData("test info value 3");
- metric1.setStartTime(0L);
- metric1.setEndTime(1L);
+ metric2.addValue(3L, (short) 3);
+ Assert.assertEquals(TimelineMetric.Type.SINGLE_VALUE, metric2.getType());
+ Assert.assertTrue(
+ metric2.getValues().values().iterator().next() instanceof Short);
+ Map<Long, Number> points = new HashMap<>();
+ points.put(4L, 4.0D);
+ points.put(5L, 5.0D);
+ try {
+ metric2.setValues(points);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Values cannot contain more than one point in"));
+ }
+ try {
+ metric2.addValues(points);
+ Assert.fail();
+ } catch (IllegalArgumentException e) {
+ Assert.assertTrue(e.getMessage().contains(
+ "Values cannot contain more than one point in"));
+ }
entity.addMetric(metric2);
+
TimelineEvent event1 = new TimelineEvent();
event1.setId("test event id 1");
event1.addInfo("test info key 1", "test info value 1");
- event1.addInfo("test info key 2", "test info value 2");
+ event1.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ event1.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ event1.getInfo().get("test info key 3") instanceof Boolean);
event1.setTimestamp(0L);
entity.addEvent(event1);
+
TimelineEvent event2 = new TimelineEvent();
event2.setId("test event id 2");
event2.addInfo("test info key 1", "test info value 1");
- event2.addInfo("test info key 2", "test info value 2");
+ event2.addInfo("test info key 2",
+ Arrays.asList("test info value 2", "test info value 3"));
+ event2.addInfo("test info key 3", true);
+ Assert.assertTrue(
+ event2.getInfo().get("test info key 3") instanceof Boolean);
event2.setTimestamp(1L);
entity.addEvent(event2);
+
entity.setCreatedTime(0L);
entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index a1707b7..68748f0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -566,7 +566,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric memoryMetric = new TimelineMetric();
memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
- memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage);
+ memoryMetric.addValue(currentTime, currentPmemUsage);
entity.addMetric(memoryMetric);
}
// if cpuUsageTotalCoresPercentage data is available
@@ -574,7 +574,7 @@ public class ContainersMonitorImpl extends AbstractService implements
ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
- cpuMetric.addTimeSeriesData(currentTime,
+ cpuMetric.addValue(currentTime,
cpuUsageTotalCoresPercentage);
entity.addMetric(cpuMetric);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84147698/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 044148a..69031a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -81,6 +81,12 @@ public class TestTimelineServiceClientIntegration {
TimelineEntity entity = new TimelineEntity();
entity.setType("test entity type");
entity.setId("test entity id");
+ TimelineMetric metric =
+ new TimelineMetric(TimelineMetric.Type.TIME_SERIES);
+ metric.setId("test metric id");
+ metric.addValue(1L, 1.0D);
+ metric.addValue(2L, 2.0D);
+ entity.addMetric(metric);
client.putEntities(entity);
client.putEntitiesAsync(entity);
} finally {
[04/50] [abbrv] hadoop git commit: YARN-3377. Fixed test failure in
TestTimelineServiceClientIntegration. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration. Contributed by Sangjin Lee.
(cherry picked from commit 04de2ceccd02edb00bd671a63e04855c132e9735)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afa99256
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afa99256
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afa99256
Branch: refs/heads/YARN-2928
Commit: afa99256694dc3d4c4c2f74e2a5a9e7c846c57a2
Parents: 63c7210
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri Mar 20 00:20:24 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:38:43 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timelineservice/collector/TimelineCollectorManager.java | 3 +--
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa99256/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 42da7bf..d9696c9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -35,6 +35,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. (Sangjin Lee
via junping_du)
+ YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration.
+ (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa99256/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 3691162..3a4515e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -239,8 +239,7 @@ public class TimelineCollectorManager extends CompositeService {
+ GenericExceptionHandler.class.getPackage().getName() + ";"
+ YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
"/*");
- timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY,
- TimelineCollectorManager.getInstance());
+ timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY, this);
timelineRestServer.start();
} catch (Exception e) {
String msg = "The per-node collector webapp failed to start.";
[28/50] [abbrv] hadoop git commit: YARN-3792. Test case failures in
TestDistributedShell and some issue fixes related to ATSV2 (Naganarasimha G R
via sjlee)
Posted by sj...@apache.org.
YARN-3792. Test case failures in TestDistributedShell and some issue fixes related to ATSV2 (Naganarasimha G R via sjlee)
(cherry picked from commit 84f37f1c7eefec6d139cbf091c50d6c06f734323)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5751abd1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5751abd1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5751abd1
Branch: refs/heads/YARN-2928
Commit: 5751abd12891799fa3e59058e736661f6abb89b6
Parents: 42ed062
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Jun 22 20:47:56 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:13 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 33 +++----
.../applications/distributedshell/Client.java | 2 +-
.../distributedshell/TestDistributedShell.java | 91 +++++++++++++-------
.../TestDistributedShellWithNodeLabels.java | 9 +-
.../client/api/impl/TimelineClientImpl.java | 8 ++
.../application/ApplicationImpl.java | 4 +-
.../monitor/ContainersMonitorImpl.java | 15 ++--
.../RMTimelineCollectorManager.java | 2 +-
.../collector/NodeTimelineCollectorManager.java | 14 ---
.../PerNodeTimelineCollectorsAuxService.java | 3 +-
.../collector/TimelineCollectorManager.java | 2 +-
11 files changed, 107 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1e85cdf..a294fc5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -35,9 +35,6 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3333. Rename TimelineAggregator etc. to TimelineCollector. (Sangjin Lee
via junping_du)
- YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration.
- (Sangjin Lee via zjshen)
-
YARN-3034. Implement RM starting its timeline collector. (Naganarasimha G R
via junping_du)
@@ -61,27 +58,15 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3551. Consolidate data model change according to the backend
implementation (Zhijie Shen via sjlee)
- YARN-3562. unit tests failures and issues found from findbug from earlier
- ATS checkins (Naganarasimha G R via sjlee)
-
YARN-3134. Implemented Phoenix timeline writer to access HBase backend. (Li
Lu via zjshen)
YARN-3529. Added mini HBase cluster and Phoenix support to timeline service
v2 unit tests. (Li Lu via zjshen)
- YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
- Sangjin Lee via junping_du)
-
YARN-3411. [Storage implementation] explore the native HBase write schema
for storage (Vrushali C via sjlee)
- YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
- test data (Vrushali C via sjlee)
-
- YARN-3721. build is broken on YARN-2928 branch due to possible dependency
- cycle (Li Lu via sjlee)
-
YARN-3044. Made RM write app, attempt and optional container lifecycle
events to timeline service v2. (Naganarasimha G R via zjshen)
@@ -100,6 +85,24 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
BUG FIXES
+ YARN-3377. Fixed test failure in TestTimelineServiceClientIntegration.
+ (Sangjin Lee via zjshen)
+
+ YARN-3562. unit tests failures and issues found from findbug from earlier
+ ATS checkins (Naganarasimha G R via sjlee)
+
+ YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
+ Sangjin Lee via junping_du)
+
+ YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
+ test data (Vrushali C via sjlee)
+
+ YARN-3721. build is broken on YARN-2928 branch due to possible dependency
+ cycle (Li Lu via sjlee)
+
+ YARN-3792. Test case failures in TestDistributedShell and some issue fixes
+ related to ATSV2 (Naganarasimha G R via sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index ff2f594..a588544 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -463,7 +463,7 @@ public class Client {
}
if (cliParser.hasOption("flow_run_id")) {
try {
- flowRunId = Long.valueOf(cliParser.getOptionValue("flow_run_id"));
+ flowRunId = Long.parseLong(cliParser.getOptionValue("flow_run_id"));
} catch (NumberFormatException e) {
throw new IllegalArgumentException(
"Flow run is not a valid long value", e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index d31f2c8..b8a7abf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.util.JarFinder;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
@@ -85,28 +86,33 @@ public class TestDistributedShell {
@Before
public void setup() throws Exception {
- setupInternal(NUM_NMS);
+ setupInternal(NUM_NMS, currTestName);
}
- protected void setupInternal(int numNodeManager) throws Exception {
-
+ protected void setupInternal(int numNodeManager, TestName testName)
+ throws Exception {
LOG.info("Starting up YARN cluster");
conf = new YarnConfiguration();
conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
conf.set("yarn.log.dir", "target");
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- // mark if we need to launch the v1 timeline server
- boolean enableATSV1 = false;
- if (!currTestName.getMethodName().toLowerCase().contains("v2")) {
+
+ if (!testName.getMethodName().toLowerCase().contains("v2")) {
// disable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
- enableATSV1 = true;
+ conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, false);
} else {
// enable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
- conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
- + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
+ conf.set(YarnConfiguration.NM_AUX_SERVICES + "."
+ + TIMELINE_AUX_SERVICE_NAME + ".class",
+ PerNodeTimelineCollectorsAuxService.class.getName());
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ false);
}
conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
@@ -120,12 +126,11 @@ public class TestDistributedShell {
conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
true);
- conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, enableATSV1);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
@@ -295,13 +300,15 @@ public class TestDistributedShell {
if (checkHostname(appReport.getHost()) && appReport.getRpcPort() == -1) {
verified = true;
}
- if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED) {
+
+ if (appReport.getYarnApplicationState() == YarnApplicationState.FINISHED
+ && appReport.getFinalApplicationStatus() != FinalApplicationStatus.UNDEFINED) {
break;
}
}
Assert.assertTrue(errorMessage, verified);
t.join();
- LOG.info("Client run completed. Result=" + result);
+ LOG.info("Client run completed for testDSShell. Result=" + result);
Assert.assertTrue(result.get());
if (!isTestingTimelineV2) {
@@ -356,9 +363,9 @@ public class TestDistributedShell {
}
}
- private void checkTimelineV2(
- boolean haveDomain, ApplicationId appId, boolean defaultFlow)
- throws Exception {
+ private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
+ boolean defaultFlow) throws Exception {
+ LOG.info("Started checkTimelineV2 ");
// For PoC check in /tmp/timeline_service_data YARN-3264
String tmpRoot =
FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
@@ -409,12 +416,29 @@ public class TestDistributedShell {
verifyEntityTypeFileExists(basePath,
TimelineEntityType.YARN_APPLICATION.toString(),
appMetricsTimestampFileName);
- verifyStringExistsSpecifiedTimes(appEntityFile,
- ApplicationMetricsConstants.CREATED_EVENT_TYPE, 1,
- "Application created event should be published atleast once");
- verifyStringExistsSpecifiedTimes(appEntityFile,
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE, 1,
- "Application finished event should be published atleast once");
+ Assert.assertEquals(
+ "Application created event should be published atleast once",
+ 1,
+ getNumOfStringOccurences(appEntityFile,
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE));
+
+ // to avoid race condition of testcase, atleast check 4 times with sleep
+ // of 500ms
+ long numOfStringOccurences = 0;
+ for (int i = 0; i < 4; i++) {
+ numOfStringOccurences =
+ getNumOfStringOccurences(appEntityFile,
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ if (numOfStringOccurences > 0) {
+ break;
+ } else {
+ Thread.sleep(500l);
+ }
+ }
+ Assert.assertEquals(
+ "Application finished event should be published atleast once",
+ 1,
+ numOfStringOccurences);
// Verify RM posting AppAttempt life cycle Events are getting published
String appAttemptMetricsTimestampFileName =
@@ -425,12 +449,17 @@ public class TestDistributedShell {
verifyEntityTypeFileExists(basePath,
TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString(),
appAttemptMetricsTimestampFileName);
- verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
- AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE, 1,
- "AppAttempt register event should be published atleast once");
- verifyStringExistsSpecifiedTimes(appAttemptEntityFile,
- AppAttemptMetricsConstants.FINISHED_EVENT_TYPE, 1,
- "AppAttempt finished event should be published atleast once");
+ Assert.assertEquals(
+ "AppAttempt register event should be published atleast once",
+ 1,
+ getNumOfStringOccurences(appAttemptEntityFile,
+ AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE));
+
+ Assert.assertEquals(
+ "AppAttempt finished event should be published atleast once",
+ 1,
+ getNumOfStringOccurences(appAttemptEntityFile,
+ AppAttemptMetricsConstants.FINISHED_EVENT_TYPE));
} finally {
FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
}
@@ -449,8 +478,7 @@ public class TestDistributedShell {
return entityFile;
}
- private void verifyStringExistsSpecifiedTimes(File entityFile,
- String searchString, long expectedNumOfTimes, String errorMsg)
+ private long getNumOfStringOccurences(File entityFile, String searchString)
throws IOException {
BufferedReader reader = null;
String strLine;
@@ -464,7 +492,7 @@ public class TestDistributedShell {
} finally {
reader.close();
}
- Assert.assertEquals(errorMsg, expectedNumOfTimes, actualCount);
+ return actualCount;
}
/**
@@ -1109,4 +1137,3 @@ public class TestDistributedShell {
return numOfWords;
}
}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
index b62b091..496817d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShellWithNodeLabels.java
@@ -30,7 +30,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
import com.google.common.collect.ImmutableMap;
@@ -40,11 +42,14 @@ public class TestDistributedShellWithNodeLabels {
static final int NUM_NMS = 2;
TestDistributedShell distShellTest;
-
+
+ @Rule
+ public TestName currTestName = new TestName();
+
@Before
public void setup() throws Exception {
distShellTest = new TestDistributedShell();
- distShellTest.setupInternal(NUM_NMS);
+ distShellTest.setupInternal(NUM_NMS,currTestName);
}
private void initializeNodeLabels() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index d30f7b1..8312b6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -383,6 +383,14 @@ public class TimelineClientImpl extends TimelineClient {
// timelineServiceAddress could haven't be initialized yet
// or stale (only for new timeline service)
int retries = pollTimelineServiceAddress(this.maxServiceRetries);
+ if (timelineServiceAddress == null) {
+ String errMessage = "TimelineClient has reached to max retry times : "
+ + this.maxServiceRetries
+ + ", but failed to fetch timeline service address. Please verify"
+ + " Timeline Auxillary Service is configured in all the NMs";
+ LOG.error(errMessage);
+ throw new YarnException(errMessage);
+ }
// timelineServiceAddress could be stale, add retry logic here.
boolean needRetry = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 9319b6a..6e87cfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -104,11 +104,11 @@ public class ApplicationImpl implements Application {
stateMachine = stateMachineFactory.make(this);
Configuration conf = context.getConf();
if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- createAndStartTimelienClient(conf);
+ createAndStartTimelineClient(conf);
}
}
- private void createAndStartTimelienClient(Configuration conf) {
+ private void createAndStartTimelineClient(Configuration conf) {
// create and start timeline client
this.timelineClient = TimelineClient.createTimelineClient(appId);
timelineClient.init(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 68748f0..79c1907 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -95,11 +95,8 @@ public class ContainersMonitorImpl extends AbstractService implements
// For posting entities in new timeline service in a non-blocking way
// TODO replace with event loop in TimelineClient.
- private static ExecutorService threadPool =
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
-
+ private static ExecutorService threadPool;
+
@Private
public static enum ContainerMetric {
CPU, MEMORY
@@ -216,6 +213,10 @@ public class ContainersMonitorImpl extends AbstractService implements
if (publishContainerMetricsToTimelineService) {
LOG.info("NodeManager has been configured to publish container " +
"metrics to Timeline Service V2.");
+ threadPool =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
} else {
LOG.warn("NodeManager has not been configured to publish container " +
"metrics to Timeline Service V2.");
@@ -270,6 +271,9 @@ public class ContainersMonitorImpl extends AbstractService implements
// TODO remove threadPool after adding non-blocking call in TimelineClient
private static void shutdownAndAwaitTermination() {
+ if (threadPool == null) {
+ return;
+ }
threadPool.shutdown();
try {
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
@@ -683,7 +687,6 @@ public class ContainersMonitorImpl extends AbstractService implements
timelineClient.putEntities(entity);
} catch (IOException|YarnException e) {
LOG.error("putEntityNonBlocking get failed: " + e);
- throw new RuntimeException(e.toString());
}
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
index 7d1b657..116bf64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -56,7 +56,7 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
if (parts.length != 2 || parts[1].isEmpty()) {
continue;
}
- switch (parts[0]) {
+ switch (parts[0].toUpperCase()) {
case TimelineUtils.FLOW_NAME_TAG_PREFIX:
collector.getTimelineEntityContext().setFlowName(parts[1]);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 31051db..851ac30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -48,21 +48,11 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
-
-/**
- *
- * It is a singleton, and instances should be obtained via
- * {@link #getInstance()}.
- *
- */
@Private
@Unstable
public class NodeTimelineCollectorManager extends TimelineCollectorManager {
private static final Log LOG =
LogFactory.getLog(NodeTimelineCollectorManager.class);
- private static final NodeTimelineCollectorManager INSTANCE =
- new NodeTimelineCollectorManager();
-
// REST server for this collector manager
private HttpServer2 timelineRestServer;
@@ -73,10 +63,6 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
- static NodeTimelineCollectorManager getInstance() {
- return INSTANCE;
- }
-
@VisibleForTesting
protected NodeTimelineCollectorManager() {
super(NodeTimelineCollectorManager.class.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 36ff5c0..3ede97a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -56,8 +56,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
private final NodeTimelineCollectorManager collectorManager;
public PerNodeTimelineCollectorsAuxService() {
- // use the same singleton
- this(NodeTimelineCollectorManager.getInstance());
+ this(new NodeTimelineCollectorManager());
}
@VisibleForTesting PerNodeTimelineCollectorsAuxService(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5751abd1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index d54715c..23ad4f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -128,7 +128,7 @@ public abstract class TimelineCollectorManager extends AbstractService {
postRemove(appId, collector);
// stop the service to do clean up
collector.stop();
- LOG.info("the collector service for " + appId + " was removed");
+ LOG.info("The collector service for " + appId + " was removed");
}
return collector != null;
}
[36/50] [abbrv] hadoop git commit: YARN-3908. Fixed bugs in
HBaseTimelineWriterImpl. Contributed by Vrushali C and Sangjin Lee.
Posted by sj...@apache.org.
YARN-3908. Fixed bugs in HBaseTimelineWriterImpl. Contributed by Vrushali C and Sangjin Lee.
(cherry picked from commit df0ec473a84871b0effd7ca6faac776210d7df09)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b3b86a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b3b86a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b3b86a2
Branch: refs/heads/YARN-2928
Commit: 5b3b86a2571c26cdfb23720508ba604e46acfba6
Parents: 57f6d06
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Jul 27 15:50:28 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:14 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../records/timelineservice/TimelineEvent.java | 4 +-
.../storage/HBaseTimelineWriterImpl.java | 18 ++++++-
.../storage/common/ColumnHelper.java | 21 ++++----
.../storage/common/ColumnPrefix.java | 7 +--
.../storage/common/Separator.java | 7 +++
.../storage/entity/EntityColumnPrefix.java | 15 ++++--
.../storage/entity/EntityTable.java | 6 ++-
.../storage/TestHBaseTimelineWriterImpl.java | 56 ++++++++++++++++++--
9 files changed, 111 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fd19320..69112b5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -115,6 +115,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3792. Test case failures in TestDistributedShell and some issue fixes
related to ATSV2 (Naganarasimha G R via sjlee)
+ YARN-3908. Fixed bugs in HBaseTimelineWriterImpl. (Vrushali C and Sangjin
+ Lee via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index 1dbf7e5..a563658 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -33,6 +33,8 @@ import java.util.Map;
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineEvent implements Comparable<TimelineEvent> {
+ public static final long INVALID_TIMESTAMP = 0L;
+
private String id;
private HashMap<String, Object> info = new HashMap<>();
private long timestamp;
@@ -83,7 +85,7 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
}
public boolean isValid() {
- return (id != null && timestamp != 0L);
+ return (id != null && timestamp != INVALID_TIMESTAMP);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 876ad6a..cd2e76e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -141,6 +141,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
te.getModifiedTime());
EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
+ Map<String, Object> info = te.getInfo();
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ EntityColumnPrefix.INFO.store(rowKey, entityTable, entry.getKey(),
+ null, entry.getValue());
+ }
+ }
}
/**
@@ -186,6 +193,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (event != null) {
String eventId = event.getId();
if (eventId != null) {
+ long eventTimestamp = event.getTimestamp();
+ // if the timestamp is not set, use the current timestamp
+ if (eventTimestamp == TimelineEvent.INVALID_TIMESTAMP) {
+ LOG.warn("timestamp is not set for event " + eventId +
+ "! Using the current timestamp");
+ eventTimestamp = System.currentTimeMillis();
+ }
Map<String, Object> eventInfo = event.getInfo();
if (eventInfo != null) {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
@@ -198,8 +212,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// convert back to string to avoid additional API on store.
String compoundColumnQualifier =
Bytes.toString(compoundColumnQualifierBytes);
- EntityColumnPrefix.METRIC.store(rowKey, entityTable,
- compoundColumnQualifier, null, info.getValue());
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifier, eventTimestamp, info.getValue());
} // for info: eventInfo
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index 6a204dc..a902924 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -113,19 +113,22 @@ public class ColumnHelper<T> {
}
/**
- * @param result from which to reads timeseries data
+ * @param result from which to reads data with timestamps
* @param columnPrefixBytes optional prefix to limit columns. If null all
* columns are returned.
+ * @param <V> the type of the values. The values will be cast into that type.
* @return the cell values at each respective time in for form
* {idA={timestamp1->value1}, idA={timestamp2->value2},
* idB={timestamp3->value3}, idC={timestamp1->value4}}
* @throws IOException
*/
- public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
- Result result, byte[] columnPrefixBytes) throws IOException {
+ @SuppressWarnings("unchecked")
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result, byte[] columnPrefixBytes)
+ throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> results =
- new TreeMap<String, NavigableMap<Long, Number>>();
+ NavigableMap<String, NavigableMap<Long, V>> results =
+ new TreeMap<String, NavigableMap<Long, V>>();
if (result != null) {
NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
@@ -157,13 +160,13 @@ public class ColumnHelper<T> {
// If this column has the prefix we want
if (columnName != null) {
- NavigableMap<Long, Number> cellResults =
- new TreeMap<Long, Number>();
+ NavigableMap<Long, V> cellResults =
+ new TreeMap<Long, V>();
NavigableMap<Long, byte[]> cells = entry.getValue();
if (cells != null) {
for (Entry<Long, byte[]> cell : cells.entrySet()) {
- Number value =
- (Number) GenericObjectMapper.read(cell.getValue());
+ V value =
+ (V) GenericObjectMapper.read(cell.getValue());
cellResults.put(cell.getKey(), value);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index 2eedea0..671c824 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -72,12 +72,13 @@ public interface ColumnPrefix<T> {
public Map<String, Object> readResults(Result result) throws IOException;
/**
- * @param result from which to reads timeseries data
+ * @param result from which to reads data with timestamps
+ * @param <V> the type of the values. The values will be cast into that type.
* @return the cell values at each respective time in for form
* {idA={timestamp1->value1}, idA={timestamp2->value2},
* idB={timestamp3->value3}, idC={timestamp1->value4}}
* @throws IOException
*/
- public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
- Result result) throws IOException;
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result) throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
index ee57890..3319419 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -90,6 +90,13 @@ public enum Separator {
}
/**
+ * @return the original value of the separator
+ */
+ public String getValue() {
+ return value;
+ }
+
+ /**
* Used to make token safe to be used with this separator without collisions.
*
* @param token
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 4459868..8b7bc3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -45,6 +45,11 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
RELATES_TO(EntityColumnFamily.INFO, "r"),
/**
+ * To store TimelineEntity info values.
+ */
+ INFO(EntityColumnFamily.INFO, "i"),
+
+ /**
* Lifecycle events for an entity
*/
EVENT(EntityColumnFamily.INFO, "e"),
@@ -92,7 +97,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
/**
* @return the column name value
*/
- private String getColumnPrefix() {
+ public String getColumnPrefix() {
return columnPrefix;
}
@@ -150,11 +155,11 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*
* @see
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
- * #readTimeseriesResults(org.apache.hadoop.hbase.client.Result)
+ * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
- public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
- Result result) throws IOException {
- return column.readTimeseriesResults(result, columnPrefixBytes);
+ public <T> NavigableMap<String, NavigableMap<Long, T>>
+ readResultsWithTimestamps(Result result) throws IOException {
+ return column.readResultsWithTimestamps(result, columnPrefixBytes);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 61f7c4c..2ae7d39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -54,7 +54,10 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEnti
* | | modified_time: | | |
* | | 1392995081012 | metricId2: | |
* | | | metricValue1 | |
- * | | r!relatesToKey: | @timestamp2 | |
+ * | | i!infoKey: | @timestamp2 | |
+ * | | infoValue | | |
+ * | | | | |
+ * | | r!relatesToKey: | | |
* | | id3?id4?id5 | | |
* | | | | |
* | | s!isRelatedToKey | | |
@@ -62,6 +65,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEnti
* | | | | |
* | | e!eventId?eventInfoKey: | | |
* | | eventInfoValue | | |
+ * | | @timestamp | | |
* | | | | |
* | | flowVersion: | | |
* | | versionValue | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b3b86a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 6abf240..31cb5d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -43,8 +43,10 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
@@ -84,6 +86,12 @@ public class TestHBaseTimelineWriterImpl {
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
// add the isRelatedToEntity info
String key = "task";
String value = "is_related_to_entity_id_here";
@@ -177,6 +185,14 @@ public class TestHBaseTimelineWriterImpl {
Long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
+ Map<String, Object> infoColumns =
+ EntityColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap.size(), infoColumns.size());
+ for (String infoItem : infoMap.keySet()) {
+ assertEquals(infoMap.get(infoItem),
+ infoColumns.get(infoItem));
+ }
+
// Remember isRelatedTo is of type Map<String, Set<String>>
for (String isRelatedToKey : isRelatedTo.keySet()) {
Object isRelatedToValue =
@@ -219,7 +235,7 @@ public class TestHBaseTimelineWriterImpl {
}
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readTimeseriesResults(result);
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
// We got metrics back
@@ -237,7 +253,7 @@ public class TestHBaseTimelineWriterImpl {
}
}
assertEquals(1, rowCount);
- assertEquals(15, colCount);
+ assertEquals(17, colCount);
} finally {
hbi.stop();
@@ -267,13 +283,18 @@ public class TestHBaseTimelineWriterImpl {
private void testAdditionalEntity() throws IOException {
TimelineEvent event = new TimelineEvent();
- event.setId("foo_event_id");
- event.setTimestamp(System.currentTimeMillis());
- event.addInfo("foo_event", "test");
+ String eventId = "foo_event_id";
+ event.setId(eventId);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
final TimelineEntity entity = new TimelineEntity();
entity.setId("attempt_1329348432655_0001_m_000008_18");
entity.setType("FOO_ATTEMPT");
+ entity.addEvent(event);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(entity);
@@ -304,6 +325,31 @@ public class TestHBaseTimelineWriterImpl {
for (Result result : scanner) {
if (result != null && !result.isEmpty()) {
rowCount++;
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ // check the events
+ NavigableMap<String, NavigableMap<Long, Object>> eventsResult =
+ EntityColumnPrefix.EVENT.readResultsWithTimestamps(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ // key name for the event
+ String valueKey = eventId + Separator.VALUES.getValue() + expKey;
+ for (Map.Entry<String, NavigableMap<Long, Object>> e :
+ eventsResult.entrySet()) {
+ // the value key must match
+ assertEquals(valueKey, e.getKey());
+ NavigableMap<Long, Object> value = e.getValue();
+ // there should be only one timestamp and value
+ assertEquals(1, value.size());
+ for (Map.Entry<Long, Object> e2: value.entrySet()) {
+ assertEquals(expTs, e2.getKey());
+ assertEquals(expVal, e2.getValue());
+ }
+ }
}
}
assertEquals(1, rowCount);
[13/50] [abbrv] hadoop git commit: YARN-3334. NM uses timeline client
to publish container metrics to new timeline service. Contributed by Junping
Du.
Posted by sj...@apache.org.
YARN-3334. NM uses timeline client to publish container metrics to new timeline service. Contributed by Junping Du.
(cherry picked from commit ae0a11167265c126f8127cce15b4b717e3a8767c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/462c48ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/462c48ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/462c48ab
Branch: refs/heads/YARN-2928
Commit: 462c48ab7c3b0b1547e021055951fd355a6c1dee
Parents: a9c81ac
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Apr 6 09:31:24 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../HierarchicalTimelineEntity.java | 4 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 8 ++
.../distributedshell/ApplicationMaster.java | 32 +++--
.../distributedshell/TestDistributedShell.java | 47 +++++--
.../client/api/impl/TimelineClientImpl.java | 11 +-
.../hadoop/yarn/server/nodemanager/Context.java | 10 +-
.../yarn/server/nodemanager/NodeManager.java | 33 ++---
.../nodemanager/NodeStatusUpdaterImpl.java | 49 +++++++-
.../collectormanager/NMCollectorService.java | 11 +-
.../application/Application.java | 3 +
.../application/ApplicationImpl.java | 26 +++-
.../monitor/ContainersMonitorImpl.java | 123 ++++++++++++++++++-
.../yarn/server/nodemanager/TestEventFlow.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 8 +-
.../BaseContainerManagerTest.java | 2 +-
.../TestContainerManagerRecovery.java | 4 +-
.../launcher/TestContainerLaunch.java | 6 +-
.../TestLocalCacheDirectoryManager.java | 2 +-
.../TestResourceLocalizationService.java | 4 +-
.../yarn/server/nodemanager/webapp/MockApp.java | 8 +-
.../webapp/TestContainerLogsPage.java | 12 +-
.../nodemanager/webapp/TestNMWebServer.java | 10 +-
.../nodemanager/webapp/TestNMWebServices.java | 2 +-
.../webapp/TestNMWebServicesApps.java | 2 +-
.../webapp/TestNMWebServicesContainers.java | 2 +-
.../resourcemanager/ResourceTrackerService.java | 20 ++-
.../collector/TimelineCollectorWebService.java | 2 +-
28 files changed, 334 insertions(+), 112 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 76fa0a8..a3a1f14 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -47,6 +47,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3374. Collector's web server should randomly bind an available port. (
Zhijie Shen via junping_du)
+ YARN-3334. NM uses timeline client to publish container metrics to new
+ timeline service. (Junping Du via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
index 01d85cf..49576de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
@@ -58,7 +58,9 @@ public abstract class HierarchicalTimelineEntity extends TimelineEntity {
// required by JAXB
@InterfaceAudience.Private
- @XmlElement(name = "children")
+ // comment out XmlElement here because it cause UnrecognizedPropertyException
+ // TODO we need a better fix
+ //@XmlElement(name = "children")
public HashMap<String, Set<String>> getChildrenJAXB() {
return children;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1ba7f36..e26b13f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2121,6 +2121,14 @@ public class YarnConfiguration extends Configuration {
}
return clusterId;
}
+
+ public static boolean systemMetricsPublisherEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
+ && conf.getBoolean(
+ YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
+ }
/* For debugging. mp configurations to system output as XML format. */
public static void main(String[] args) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index e107731..e27c947 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -314,19 +314,8 @@ public class ApplicationMaster {
}
appMaster.run();
result = appMaster.finish();
-
- threadPool.shutdown();
-
- while (!threadPool.isTerminated()) { // wait for all posting thread to finish
- try {
- if (!threadPool.awaitTermination(30, TimeUnit.SECONDS)) {
- threadPool.shutdownNow(); // send interrupt to hurry them along
- }
- } catch (InterruptedException e) {
- LOG.warn("Timeline client service stop interrupted!");
- break;
- }
- }
+
+ shutdownAndAwaitTermination();
} catch (Throwable t) {
LOG.fatal("Error running ApplicationMaster", t);
LogManager.shutdown();
@@ -340,6 +329,23 @@ public class ApplicationMaster {
System.exit(2);
}
}
+
+ //TODO remove threadPool after adding non-blocking call in TimelineClient
+ private static void shutdownAndAwaitTermination() {
+ threadPool.shutdown();
+ try {
+ // Wait a while for existing tasks to terminate
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
+ threadPool.shutdownNow();
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
+ LOG.error("ThreadPool did not terminate");
+ }
+ } catch (InterruptedException ie) {
+ threadPool.shutdownNow();
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ }
/**
* Dump out contents of $CWD and the environment to stdout for debugging
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 1de3b68..daaad7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -48,13 +48,17 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ProcfsBasedProcessTree;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -105,6 +109,16 @@ public class TestDistributedShell {
}
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ // Enable ContainersMonitorImpl
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR,
+ LinuxResourceCalculatorPlugin.class.getName());
+ conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE,
+ ProcfsBasedProcessTree.class.getName());
+ conf.setBoolean(YarnConfiguration.NM_PMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.NM_VMEM_CHECK_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_CONTROL_RESOURCE_MONITORING,
+ true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
if (yarnCluster == null) {
yarnCluster =
@@ -349,15 +363,14 @@ public class TestDistributedShell {
File tmpRootFolder = new File(tmpRoot);
try {
Assert.assertTrue(tmpRootFolder.isDirectory());
-
- // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
- String outputDirApp = tmpRoot +
+ String basePath = tmpRoot +
YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
UserGroupInformation.getCurrentUser().getShortUserName() +
(defaultFlow ? "/" +
TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
- "/0/" : "/test_flow_id/12345678/") +
- appId.toString() + "/DS_APP_ATTEMPT/";
+ "/0/" : "/test_flow_id/12345678/") + appId.toString();
+ // for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
+ String outputDirApp = basePath + "/DS_APP_ATTEMPT/";
File entityFolder = new File(outputDirApp);
Assert.assertTrue(entityFolder.isDirectory());
@@ -370,13 +383,7 @@ public class TestDistributedShell {
File appAttemptFile = new File(appAttemptFileName);
Assert.assertTrue(appAttemptFile.exists());
- String outputDirContainer = tmpRoot +
- YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
- UserGroupInformation.getCurrentUser().getShortUserName() +
- (defaultFlow ? "/" +
- TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
- "/0/" : "/test_flow_id/12345678/") +
- appId.toString() + "/DS_CONTAINER/";
+ String outputDirContainer = basePath + "/DS_CONTAINER/";
File containerFolder = new File(outputDirContainer);
Assert.assertTrue(containerFolder.isDirectory());
@@ -388,6 +395,22 @@ public class TestDistributedShell {
Assert.assertTrue(containerFile.exists());
String appTimeStamp = appId.getClusterTimestamp() + "_" + appId.getId()
+ "_";
+
+ // Verify NM posting container metrics info.
+ String outputDirContainerMetrics = basePath + "/" +
+ TimelineEntityType.YARN_CONTAINER + "/";
+ File containerMetricsFolder = new File(outputDirContainerMetrics);
+ Assert.assertTrue(containerMetricsFolder.isDirectory());
+
+ String containerMetricsTimestampFileName = "container_"
+ + appId.getClusterTimestamp() + "_000" + appId.getId()
+ + "_01_000001.thist";
+ String containerMetricsFileName = outputDirContainerMetrics +
+ containerMetricsTimestampFileName;
+
+ File containerMetricsFile = new File(containerMetricsFileName);
+ Assert.assertTrue(containerMetricsFile.exists());
+
} finally {
FileUtils.deleteDirectory(tmpRootFolder.getParentFile());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index c722edf..d30f7b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -447,14 +447,11 @@ public class TimelineClientImpl extends TimelineClient {
}
if (resp == null ||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
- String msg =
- "Failed to get the response from the timeline server.";
+ String msg = "Response from the timeline server is " +
+ ((resp == null) ? "null":
+ "not successful," + " HTTP error code: " + resp.getStatus()
+ + ", Server response:\n" + resp.getEntity(String.class));
LOG.error(msg);
- if (LOG.isDebugEnabled() && resp != null) {
- String output = resp.getEntity(String.class);
- LOG.debug("HTTP error code: " + resp.getStatus()
- + " Server response:\n" + output);
- }
throw new YarnException(msg);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index fed7113..179dd92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -22,6 +22,7 @@ import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.ContainerManagementProtocol;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -66,13 +67,6 @@ public interface Context {
*/
Map<ApplicationId, String> getRegisteredCollectors();
- /**
- * Return the known collectors which get from RM for all active applications
- * running on this NM.
- * @return known collectors.
- */
- Map<ApplicationId, String> getKnownCollectors();
-
ConcurrentMap<ContainerId, Container> getContainers();
NMContainerTokenSecretManager getContainerTokenSecretManager();
@@ -92,6 +86,8 @@ public interface Context {
NMStateStoreService getNMStateStore();
boolean getDecommissioned();
+
+ Configuration getConf();
void setDecommissioned(boolean isDecommissioned);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 441388b..697b706 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -188,9 +188,9 @@ public class NodeManager extends CompositeService
protected NMContext createNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
- NMStateStoreService stateStore) {
+ NMStateStoreService stateStore, Configuration conf) {
return new NMContext(containerTokenSecretManager, nmTokenSecretManager,
- dirsHandler, aclsManager, stateStore);
+ dirsHandler, aclsManager, stateStore, conf);
}
protected void doSecureLogin() throws IOException {
@@ -312,7 +312,7 @@ public class NodeManager extends CompositeService
addService(nodeHealthChecker);
this.context = createNMContext(containerTokenSecretManager,
- nmTokenSecretManager, nmStore);
+ nmTokenSecretManager, nmStore, conf);
nodeLabelsProvider = createNodeLabelsProvider(conf);
@@ -439,6 +439,9 @@ public class NodeManager extends CompositeService
public static class NMContext implements Context {
private NodeId nodeId = null;
+
+ private Configuration conf = null;
+
protected final ConcurrentMap<ApplicationId, Application> applications =
new ConcurrentHashMap<ApplicationId, Application>();
@@ -451,9 +454,6 @@ public class NodeManager extends CompositeService
protected Map<ApplicationId, String> registeredCollectors =
new ConcurrentHashMap<ApplicationId, String>();
- protected Map<ApplicationId, String> knownCollectors =
- new ConcurrentHashMap<ApplicationId, String>();
-
private final NMContainerTokenSecretManager containerTokenSecretManager;
private final NMTokenSecretManagerInNM nmTokenSecretManager;
private ContainerManagementProtocol containerManager;
@@ -471,7 +471,7 @@ public class NodeManager extends CompositeService
public NMContext(NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
- NMStateStoreService stateStore) {
+ NMStateStoreService stateStore, Configuration conf) {
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
this.dirsHandler = dirsHandler;
@@ -482,6 +482,7 @@ public class NodeManager extends CompositeService
this.stateStore = stateStore;
this.logAggregationReportForApps = new ConcurrentLinkedQueue<
LogAggregationReport>();
+ this.conf = conf;
}
/**
@@ -501,6 +502,11 @@ public class NodeManager extends CompositeService
public ConcurrentMap<ApplicationId, Application> getApplications() {
return this.applications;
}
+
+ @Override
+ public Configuration getConf() {
+ return this.conf;
+ }
@Override
public ConcurrentMap<ContainerId, Container> getContainers() {
@@ -597,19 +603,6 @@ public class NodeManager extends CompositeService
public void addRegisteredCollectors(
Map<ApplicationId, String> newRegisteredCollectors) {
this.registeredCollectors.putAll(newRegisteredCollectors);
- // Update to knownCollectors as well so it can immediately be consumed by
- // this NM's TimelineClient.
- this.knownCollectors.putAll(newRegisteredCollectors);
- }
-
- @Override
- public Map<ApplicationId, String> getKnownCollectors() {
- return this.knownCollectors;
- }
-
- public void addKnownCollectors(
- Map<ApplicationId, String> knownCollectors) {
- this.knownCollectors.putAll(knownCollectors);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index c19a7d3..16d8b2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -76,6 +77,7 @@ import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
@@ -766,10 +768,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
((NMContext) context)
.setSystemCrendentialsForApps(parseCredentials(systemCredentials));
}
-
- Map<ApplicationId, String> knownCollectors =
- response.getAppCollectorsMap();
- ((NodeManager.NMContext)context).addKnownCollectors(knownCollectors);
+ if (YarnConfiguration.systemMetricsPublisherEnabled(context.getConf())) {
+ updateTimelineClientsAddress(response);
+ }
} catch (ConnectException e) {
//catch and throw the exception if tried MAX wait time to connect RM
@@ -796,6 +797,46 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
}
}
+ /**
+ * Caller should take care of sending non null nodelabels for both
+ * arguments
+ *
+ * @param nodeLabelsNew
+ * @param nodeLabelsOld
+ * @return if the New node labels are diff from the older one.
+ */
+ private boolean areNodeLabelsUpdated(Set<NodeLabel> nodeLabelsNew,
+ Set<NodeLabel> nodeLabelsOld) {
+ if (nodeLabelsNew.size() != nodeLabelsOld.size()
+ || !nodeLabelsOld.containsAll(nodeLabelsNew)) {
+ return true;
+ }
+ return false;
+ }
+
+ private void updateTimelineClientsAddress(
+ NodeHeartbeatResponse response) {
+ Set<Map.Entry<ApplicationId, String>> rmKnownCollectors =
+ response.getAppCollectorsMap().entrySet();
+ for (Map.Entry<ApplicationId, String> entry : rmKnownCollectors) {
+ ApplicationId appId = entry.getKey();
+ String collectorAddr = entry.getValue();
+
+ // Only handle applications running on local node.
+ // Not include apps with timeline collectors running in local
+ Application application = context.getApplications().get(appId);
+ if (application != null &&
+ !context.getRegisteredCollectors().containsKey(appId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sync a new collector address: " + collectorAddr +
+ " for application: " + appId + " from RM.");
+ }
+ TimelineClient client = application.getTimelineClient();
+ client.setTimelineServiceAddress(collectorAddr);
+ }
+ }
+ }
+
private void updateMasterKeys(NodeHeartbeatResponse response) {
// See if the master-key has rolled over
MasterKey updatedMasterKey = response.getContainerTokenMasterKey();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index 6ccea84..f37be23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -103,7 +104,15 @@ public class NMCollectorService extends CompositeService implements
Map<ApplicationId, String> newCollectorsMap =
new HashMap<ApplicationId, String>();
for (AppCollectorsMap collector : newCollectorsList) {
- newCollectorsMap.put(collector.getApplicationId(), collector.getCollectorAddr());
+ ApplicationId appId = collector.getApplicationId();
+ String collectorAddr = collector.getCollectorAddr();
+ newCollectorsMap.put(appId, collectorAddr);
+ // set registered collector address to TimelineClient.
+ if (YarnConfiguration.systemMetricsPublisherEnabled(context.getConf())) {
+ TimelineClient client =
+ context.getApplications().get(appId).getTimelineClient();
+ client.setTimelineServiceAddress(collectorAddr);
+ }
}
((NodeManager.NMContext)context).addRegisteredCollectors(newCollectorsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
index decd17d..0c95193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
@@ -22,6 +22,7 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -38,5 +39,7 @@ public interface Application extends EventHandler<ApplicationEvent> {
String getFlowId();
String getFlowRunId();
+
+ TimelineClient getTimelineClient();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 67604d9..b0ab550 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -28,12 +28,15 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServicesEvent;
@@ -73,6 +76,7 @@ public class ApplicationImpl implements Application {
private final ReadLock readLock;
private final WriteLock writeLock;
private final Context context;
+ private TimelineClient timelineClient;
private static final Log LOG = LogFactory.getLog(ApplicationImpl.class);
@@ -96,6 +100,17 @@ public class ApplicationImpl implements Application {
readLock = lock.readLock();
writeLock = lock.writeLock();
stateMachine = stateMachineFactory.make(this);
+ Configuration conf = context.getConf();
+ if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ createAndStartTimelienClient(conf);
+ }
+ }
+
+ private void createAndStartTimelienClient(Configuration conf) {
+ // create and start timeline client
+ this.timelineClient = TimelineClient.createTimelineClient(appId);
+ timelineClient.init(conf);
+ timelineClient.start();
}
@Override
@@ -107,6 +122,11 @@ public class ApplicationImpl implements Application {
public ApplicationId getAppId() {
return appId;
}
+
+ @Override
+ public TimelineClient getTimelineClient() {
+ return timelineClient;
+ }
@Override
public ApplicationState getApplicationState() {
@@ -433,7 +453,11 @@ public class ApplicationImpl implements Application {
// TODO check we remove related collectors info in failure cases
// (YARN-3038)
app.context.getRegisteredCollectors().remove(app.getAppId());
- app.context.getKnownCollectors().remove(app.getAppId());
+ // stop timelineClient when application get finished.
+ TimelineClient timelineClient = app.getTimelineClient();
+ if (timelineClient != null) {
+ timelineClient.stop();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index afb51ad..a1707b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -18,32 +18,43 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
-import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainersMonitorImpl extends AbstractService implements
ContainersMonitor {
@@ -74,11 +85,25 @@ public class ContainersMonitorImpl extends AbstractService implements
private boolean pmemCheckEnabled;
private boolean vmemCheckEnabled;
private boolean containersMonitorEnabled;
+
+ private boolean publishContainerMetricsToTimelineService;
private long maxVCoresAllottedForContainers;
private static final long UNKNOWN_MEMORY_LIMIT = -1L;
private int nodeCpuPercentageForYARN;
+
+ // For posting entities in new timeline service in a non-blocking way
+ // TODO replace with event loop in TimelineClient.
+ private static ExecutorService threadPool =
+ Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
+
+ @Private
+ public static enum ContainerMetric {
+ CPU, MEMORY
+ }
private ResourceUtilization containersUtilization;
@@ -184,6 +209,18 @@ public class ContainersMonitorImpl extends AbstractService implements
1) + "). Thrashing might happen.");
}
}
+
+ publishContainerMetricsToTimelineService =
+ YarnConfiguration.systemMetricsPublisherEnabled(conf);
+
+ if (publishContainerMetricsToTimelineService) {
+ LOG.info("NodeManager has been configured to publish container " +
+ "metrics to Timeline Service V2.");
+ } else {
+ LOG.warn("NodeManager has not been configured to publish container " +
+ "metrics to Timeline Service V2.");
+ }
+
super.serviceInit(conf);
}
@@ -225,8 +262,27 @@ public class ContainersMonitorImpl extends AbstractService implements
;
}
}
+
+ shutdownAndAwaitTermination();
+
super.serviceStop();
}
+
+ // TODO remove threadPool after adding non-blocking call in TimelineClient
+ private static void shutdownAndAwaitTermination() {
+ threadPool.shutdown();
+ try {
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
+ threadPool.shutdownNow();
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
+ LOG.error("ThreadPool did not terminate");
+ }
+ } catch (InterruptedException ie) {
+ threadPool.shutdownNow();
+ // Preserve interrupt status
+ Thread.currentThread().interrupt();
+ }
+ }
private static class ProcessTreeInfo {
private ContainerId containerId;
@@ -410,10 +466,14 @@ public class ContainersMonitorImpl extends AbstractService implements
long cpuUsageTotalCoresByAllContainers = 0;
for (Iterator<Map.Entry<ContainerId, ProcessTreeInfo>> it =
trackingContainers.entrySet().iterator(); it.hasNext();) {
-
+
Map.Entry<ContainerId, ProcessTreeInfo> entry = it.next();
ContainerId containerId = entry.getKey();
ProcessTreeInfo ptInfo = entry.getValue();
+
+ ContainerEntity entity = new ContainerEntity();
+ entity.setId(containerId.toString());
+
try {
String pId = ptInfo.getPID();
@@ -428,7 +488,8 @@ public class ContainersMonitorImpl extends AbstractService implements
+ " for the first time");
ResourceCalculatorProcessTree pt =
- ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(pId, processTreeClass, conf);
+ ResourceCalculatorProcessTree.getResourceCalculatorProcessTree(
+ pId, processTreeClass, conf);
ptInfo.setPid(pId);
ptInfo.setProcessTree(pt);
@@ -456,13 +517,15 @@ public class ContainersMonitorImpl extends AbstractService implements
pTree.updateProcessTree(); // update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
+ long currentTime = System.currentTimeMillis();
+
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300% and
// cpuUsageTotalCoresPercentage should be 50%
float cpuUsagePercentPerCore = pTree.getCpuUsagePercent();
float cpuUsageTotalCoresPercentage = cpuUsagePercentPerCore /
resourceCalculatorPlugin.getNumProcessors();
-
+
// Multiply by 1000 to avoid losing data when converting to int
int milliVcoresUsed = (int) (cpuUsageTotalCoresPercentage * 1000
* maxVCoresAllottedForContainers /nodeCpuPercentageForYARN);
@@ -497,6 +560,26 @@ public class ContainersMonitorImpl extends AbstractService implements
((int)cpuUsagePercentPerCore, milliVcoresUsed);
}
+ if (publishContainerMetricsToTimelineService) {
+ // if currentPmemUsage data is available
+ if (currentPmemUsage !=
+ ResourceCalculatorProcessTree.UNAVAILABLE) {
+ TimelineMetric memoryMetric = new TimelineMetric();
+ memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
+ memoryMetric.addTimeSeriesData(currentTime, currentPmemUsage);
+ entity.addMetric(memoryMetric);
+ }
+ // if cpuUsageTotalCoresPercentage data is available
+ if (cpuUsageTotalCoresPercentage !=
+ ResourceCalculatorProcessTree.UNAVAILABLE) {
+ TimelineMetric cpuMetric = new TimelineMetric();
+ cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
+ cpuMetric.addTimeSeriesData(currentTime,
+ cpuUsageTotalCoresPercentage);
+ entity.addMetric(cpuMetric);
+ }
+ }
+
boolean isMemoryOverLimit = false;
String msg = "";
int containerExitStatus = ContainerExitStatus.INVALID;
@@ -551,10 +634,23 @@ public class ContainersMonitorImpl extends AbstractService implements
it.remove();
LOG.info("Removed ProcessTree with root " + pId);
}
+
} catch (Exception e) {
// Log the exception and proceed to the next container.
- LOG.warn("Uncaught exception in ContainerMemoryManager "
- + "while managing memory of " + containerId, e);
+ LOG.warn("Uncaught exception in ContainersMonitorImpl "
+ + "while monitoring resource of " + containerId, e);
+ }
+
+ if (publishContainerMetricsToTimelineService) {
+ try {
+ TimelineClient timelineClient = context.getApplications().get(
+ containerId.getApplicationAttemptId().getApplicationId()).
+ getTimelineClient();
+ putEntityWithoutBlocking(timelineClient, entity);
+ } catch (Exception e) {
+ LOG.error("Exception in ContainersMonitorImpl in putting " +
+ "resource usage metrics to timeline service.", e);
+ }
}
}
if (LOG.isDebugEnabled()) {
@@ -578,6 +674,21 @@ public class ContainersMonitorImpl extends AbstractService implements
}
}
}
+
+ private void putEntityWithoutBlocking(final TimelineClient timelineClient,
+ final TimelineEntity entity) {
+ Runnable publishWrapper = new Runnable() {
+ public void run() {
+ try {
+ timelineClient.putEntities(entity);
+ } catch (IOException|YarnException e) {
+ LOG.error("putEntityNonBlocking get failed: " + e);
+ throw new RuntimeException(e.toString());
+ }
+ }
+ };
+ threadPool.execute(publishWrapper);
+ }
private String formatErrorMessage(String memTypeExceeded,
long currentVmemUsage, long vmemLimit,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
index c878e5c..6ac9c87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
@@ -81,7 +81,7 @@ public class TestEventFlow {
Context context = new NMContext(new NMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInNM(), null, null,
- new NMNullStateStoreService()) {
+ new NMNullStateStoreService(), conf) {
@Override
public int getHttpPort() {
return 1234;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index a9ef72f..7d895f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -1438,9 +1438,9 @@ public class TestNodeStatusUpdater {
protected NMContext createNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInNM nmTokenSecretManager,
- NMStateStoreService store) {
+ NMStateStoreService store, Configuration conf) {
return new MyNMContext(containerTokenSecretManager,
- nmTokenSecretManager);
+ nmTokenSecretManager, conf);
}
};
@@ -1609,9 +1609,9 @@ public class TestNodeStatusUpdater {
public MyNMContext(
NMContainerTokenSecretManager containerTokenSecretManager,
- NMTokenSecretManagerInNM nmTokenSecretManager) {
+ NMTokenSecretManagerInNM nmTokenSecretManager, Configuration conf) {
super(containerTokenSecretManager, nmTokenSecretManager, null, null,
- new NMNullStateStoreService());
+ new NMNullStateStoreService(), conf);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index a8e723d..1c0d4fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -108,7 +108,7 @@ public abstract class BaseContainerManagerTest {
protected Configuration conf = new YarnConfiguration();
protected Context context = new NMContext(new NMContainerTokenSecretManager(
conf), new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), new NMNullStateStoreService()) {
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf) {
public int getHttpPort() {
return HTTP_PORT;
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 781950e..9c3109d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Mockito.isA;
+import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy;
@@ -310,7 +310,7 @@ public class TestContainerManagerRecovery {
NMStateStoreService stateStore) {
NMContext context = new NMContext(new NMContainerTokenSecretManager(
conf), new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), stateStore);
+ new ApplicationACLsManager(conf), stateStore, conf);
// simulate registration with RM
MasterKey masterKey = new MasterKeyPBImpl();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index ea6bb1d..ec61470 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -81,8 +81,8 @@ import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.ExitCode;
-import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -90,10 +90,10 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch.ShellScriptBuilder;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.security.NMTokenSecretManagerInNM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
@@ -109,7 +109,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
protected Context distContext = new NMContext(new NMContainerTokenSecretManager(
conf), new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), new NMNullStateStoreService()) {
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf) {
public int getHttpPort() {
return HTTP_PORT;
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
index 9e08b7f..be1f343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalCacheDirectoryManager.java
@@ -81,7 +81,7 @@ public class TestLocalCacheDirectoryManager {
NMContext nmContext =
new NMContext(new NMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf);
ResourceLocalizationService service =
new ResourceLocalizationService(null, null, null, null, nmContext);
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index c515506..b761782 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -179,7 +179,7 @@ public class TestResourceLocalizationService {
conf.set(YarnConfiguration.NM_LOG_DIRS, logDir);
nmContext = new NMContext(new NMContainerTokenSecretManager(
conf), new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf);
}
@After
@@ -2064,7 +2064,7 @@ public class TestResourceLocalizationService {
NMContext nmContext =
new NMContext(new NMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInNM(), null,
- new ApplicationACLsManager(conf), stateStore);
+ new ApplicationACLsManager(conf), stateStore, conf);
ResourceLocalizationService rawService =
new ResourceLocalizationService(dispatcher, exec, delService,
dirsHandler, nmContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
index 35b95ee..2ee572b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
@@ -24,6 +24,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
@@ -41,7 +42,7 @@ public class MockApp implements Application {
Application app;
String flowId;
String flowRunId;
-
+ TimelineClient timelineClient = null;
public MockApp(int uniqId) {
this("mockUser", 1234, uniqId);
@@ -87,4 +88,9 @@ public class MockApp implements Application {
public String getFlowRunId() {
return flowRunId;
}
+
+ @Override
+ public TimelineClient getTimelineClient() {
+ return timelineClient;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 84e42fc..cd62c92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.junit.Assume.assumeTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
import java.io.BufferedOutputStream;
import java.io.File;
@@ -47,10 +47,9 @@ import org.apache.hadoop.util.NodeHealthScriptRunner;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationAttemptIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ContainerIdPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -63,6 +62,7 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
@@ -96,7 +96,7 @@ public class TestContainerLogsPage {
healthChecker.init(conf);
LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
- new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf);
// Add an application and the corresponding containers
RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
String user = "nobody";
@@ -136,7 +136,7 @@ public class TestContainerLogsPage {
when(dirsHandlerForFullDisk.getLogDirsForRead()).
thenReturn(Arrays.asList(new String[] {absLogDir.getAbsolutePath()}));
nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk,
- new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf);
nmContext.getApplications().put(appId, app);
container.setState(ContainerState.RUNNING);
nmContext.getContainers().put(container1, container);
@@ -158,7 +158,7 @@ public class TestContainerLogsPage {
LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
dirsHandler.init(conf);
NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler,
- new ApplicationACLsManager(conf), new NMNullStateStoreService());
+ new ApplicationACLsManager(conf), new NMNullStateStoreService(), conf);
// Add an application and the corresponding containers
String user = "nobody";
long clusterTimeStamp = 1234;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
index e1845c7..ed94fb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
@@ -86,8 +86,9 @@ public class TestNMWebServer {
}
private int startNMWebAppServer(String webAddr) {
+ Configuration conf = new Configuration();
Context nmContext = new NodeManager.NMContext(null, null, null, null,
- null);
+ null, conf);
ResourceView resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -110,7 +111,7 @@ public class TestNMWebServer {
return true;
}
};
- Configuration conf = new Configuration();
+
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
@@ -149,8 +150,9 @@ public class TestNMWebServer {
@Test
public void testNMWebApp() throws IOException, YarnException {
+ Configuration conf = new Configuration();
Context nmContext = new NodeManager.NMContext(null, null, null, null,
- null);
+ null, conf);
ResourceView resourceView = new ResourceView() {
@Override
public long getVmemAllocatedForContainers() {
@@ -173,7 +175,7 @@ public class TestNMWebServer {
return true;
}
};
- Configuration conf = new Configuration();
+
conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 2ac9c22..476f3df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -111,7 +111,7 @@ public class TestNMWebServices extends JerseyTestBase {
healthChecker.init(conf);
aclsManager = new ApplicationACLsManager(conf);
nmContext = new NodeManager.NMContext(null, null, dirsHandler,
- aclsManager, null);
+ aclsManager, null, conf);
NodeId nodeId = NodeId.newInstance("testhost.foo.com", 8042);
((NodeManager.NMContext)nmContext).setNodeId(nodeId);
resourceView = new ResourceView() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
index e274abb..85ae1c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
@@ -104,7 +104,7 @@ public class TestNMWebServicesApps extends JerseyTestBase {
dirsHandler = healthChecker.getDiskHandler();
aclsManager = new ApplicationACLsManager(conf);
nmContext = new NodeManager.NMContext(null, null, dirsHandler,
- aclsManager, null);
+ aclsManager, null, conf);
NodeId nodeId = NodeId.newInstance("testhost.foo.com", 9999);
((NodeManager.NMContext)nmContext).setNodeId(nodeId);
resourceView = new ResourceView() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
index 3bfd440..db78116 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
@@ -131,7 +131,7 @@ public class TestNMWebServicesContainers extends JerseyTestBase {
dirsHandler = healthChecker.getDiskHandler();
aclsManager = new ApplicationACLsManager(conf);
nmContext = new NodeManager.NMContext(null, null, dirsHandler,
- aclsManager, null) {
+ aclsManager, null, conf) {
public NodeId getNodeId() {
return NodeId.newInstance("testhost.foo.com", 8042);
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index e98fefc..ad38a71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -508,18 +508,16 @@ public class ResourceTrackerService extends AbstractService implements
Map<ApplicationId, String> liveAppCollectorsMap = new
ConcurrentHashMap<ApplicationId, String>();
Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps();
- for (ApplicationId appId : liveApps) {
- String appCollectorAddr = rmApps.get(appId).getCollectorAddr();
- if (appCollectorAddr != null) {
- liveAppCollectorsMap.put(appId, appCollectorAddr);
- } else {
- // Log a debug info if collector address is not found.
- if (LOG.isDebugEnabled()) {
- LOG.debug("Collector for applicaton: " + appId +
- " hasn't registered yet!");
- }
- }
+ // Set collectors for all apps now.
+ // TODO set collectors for only active apps running on NM (liveApps cannot be
+ // used for this case)
+ for (Map.Entry<ApplicationId, RMApp> rmApp : rmApps.entrySet()) {
+ ApplicationId appId = rmApp.getKey();
+ String appCollectorAddr = rmApp.getValue().getCollectorAddr();
+ if (appCollectorAddr != null) {
+ liveAppCollectorsMap.put(appId, appCollectorAddr);
}
+ }
response.setAppCollectorsMap(liveAppCollectorsMap);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/462c48ab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index 0f51656..edec0d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -135,7 +135,7 @@ public class TimelineCollectorWebService {
}
TimelineCollector collector = getCollector(req, appId);
if (collector == null) {
- LOG.error("Application not found");
+ LOG.error("Application: "+ appId + " is not found");
throw new NotFoundException(); // different exception?
}
collector.putEntities(entities, callerUgi);
[48/50] [abbrv] hadoop git commit: YARN-4064. build is broken at
TestHBaseTimelineWriterImpl.java (sjlee)
Posted by sj...@apache.org.
YARN-4064. build is broken at TestHBaseTimelineWriterImpl.java (sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e979f308
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e979f308
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e979f308
Branch: refs/heads/YARN-2928
Commit: e979f30831fc87f4c535df045527b466fe584ace
Parents: 89f8fd2
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Aug 19 17:46:03 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:52:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../storage/TestHBaseTimelineStorage.java | 770 +++++++++++++++++++
.../storage/TestHBaseTimelineWriterImpl.java | 770 -------------------
3 files changed, 772 insertions(+), 770 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e979f308/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cd7f849..0041f7f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -139,6 +139,8 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3984. Adjusted the event column key schema and avoided missing empty
event. (Vrushali C via zjshen)
+ YARN-4064. build is broken at TestHBaseTimelineWriterImpl.java (sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e979f308/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
new file mode 100644
index 0000000..2875e01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -0,0 +1,770 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+/**
+ * Various tests to test writing entities to HBase and reading them back from
+ * it.
+ *
+ * It uses a single HBase mini-cluster for all tests which is a little more
+ * realistic, and helps test correctness in the presence of other data.
+ *
+ * Each test uses a different cluster name to be able to handle its own data
+ * even if other records exist in the table. Use a different cluster name if
+ * you add a new test.
+ */
+public class TestHBaseTimelineStorage {
+
+ private static HBaseTestingUtility util;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ new EntityTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new AppToFlowTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new ApplicationTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ }
+
+ @Test
+ public void testWriteApplicationToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ ApplicationEntity entity = new ApplicationEntity();
+ String id = "hello";
+ entity.setId(id);
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_write_app";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ hbi.write(cluster, user, flow, flowVersion, runid, id, te);
+ hbi.stop();
+
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+ assertEquals(16, result.size());
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ id));
+
+ // check info column family
+ String id1 = ApplicationColumn.ID.readResult(result).toString();
+ assertEquals(id, id1);
+
+ Number val =
+ (Number) ApplicationColumn.CREATED_TIME.readResult(result);
+ long cTime1 = val.longValue();
+ assertEquals(cTime1, cTime);
+
+ val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
+ long mTime1 = val.longValue();
+ assertEquals(mTime1, mTime);
+
+ Map<String, Object> infoColumns =
+ ApplicationColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap, infoColumns);
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ ApplicationColumnPrefix.RELATES_TO.readResult(result,
+ relatesToKey).toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ ApplicationColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf, configColumns);
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ assertEquals(metricValues, metricMap);
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ id, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
+ e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
+ }
+
+ @Test
+ public void testWriteEntityToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello";
+ String type = "world";
+ entity.setId(id);
+ entity.setType(type);
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_write_entity";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.stop();
+
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ s.setStartRow(startRow);
+ s.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ int colCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+ colCount += result.size();
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ // check info column family
+ String id1 = EntityColumn.ID.readResult(result).toString();
+ assertEquals(id, id1);
+
+ String type1 = EntityColumn.TYPE.readResult(result).toString();
+ assertEquals(type, type1);
+
+ Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
+ long cTime1 = val.longValue();
+ assertEquals(cTime1, cTime);
+
+ val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
+ long mTime1 = val.longValue();
+ assertEquals(mTime1, mTime);
+
+ Map<String, Object> infoColumns =
+ EntityColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap, infoColumns);
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ EntityColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ EntityColumnPrefix.RELATES_TO.readResult(result, relatesToKey)
+ .toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ EntityColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf, configColumns);
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ assertEquals(metricValues, metricMap);
+ }
+ }
+ assertEquals(1, rowCount);
+ assertEquals(17, colCount);
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(type, e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
+ }
+
+ private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
+ String flow, long runid, String appName, TimelineEntity te) {
+
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ assertTrue(rowKeyComponents.length == 7);
+ assertEquals(user, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
+ assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
+ return true;
+ }
+
+ private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
+ String user, String flow, long runid, String appName) {
+
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
+
+ assertTrue(rowKeyComponents.length == 5);
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(user, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ return true;
+ }
+
+ @Test
+ public void testEvents() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
+ event.setId(eventId);
+ long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+
+ final TimelineEntity entity = new ApplicationEntity();
+ entity.setId(ApplicationId.newInstance(0, 1).toString());
+ entity.addEvent(event);
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_events";
+ String user = "user2";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, appName);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ appName));
+
+ Map<?, Object> eventsResult =
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
+ assertEquals(expKey, Bytes.toString(karr[2]));
+ Object value = e.getValue();
+ // there should be only one timestamp and value
+ assertEquals(expVal, value.toString());
+ }
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertNotNull(e2);
+ assertEquals(e1, e2);
+ assertEquals(1, es1.size());
+ assertEquals(1, es2.size());
+ assertEquals(es1, es2);
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertEquals(1, info.size());
+ for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
+ assertEquals(expKey, infoEntry.getKey());
+ assertEquals(expVal, infoEntry.getValue());
+ }
+ }
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
+ }
+
+ @Test
+ public void testEventsWithEmptyInfo() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ String eventId = "foo_event_id";
+ event.setId(eventId);
+ long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId("attempt_1329348432655_0001_m_000008_18");
+ entity.setType("FOO_ATTEMPT");
+ entity.addEvent(event);
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_empty_eventkey";
+ String user = "user_emptyeventkey";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ s.setStartRow(startRow);
+ s.addFamily(EntityColumnFamily.INFO.getBytes());
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ Map<?, Object> eventsResult =
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs),
+ Bytes.toLong(karr[1]));
+ // key must be empty
+ assertEquals(0, karr[2].length);
+ Object value = e.getValue();
+ // value should be empty
+ assertEquals("", value.toString());
+ }
+ }
+ }
+ assertEquals(1, rowCount);
+
+ // read the timeline entity using the reader this time
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertTrue(info == null || info.isEmpty());
+ }
+ } finally {
+ hbi.stop();
+ hbi.close();
+ hbr.stop();;
+ hbr.close();
+ }
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e979f308/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
deleted file mode 100644
index 2875e01..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ /dev/null
@@ -1,770 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-/**
- * Various tests to test writing entities to HBase and reading them back from
- * it.
- *
- * It uses a single HBase mini-cluster for all tests which is a little more
- * realistic, and helps test correctness in the presence of other data.
- *
- * Each test uses a different cluster name to be able to handle its own data
- * even if other records exist in the table. Use a different cluster name if
- * you add a new test.
- */
-public class TestHBaseTimelineStorage {
-
- private static HBaseTestingUtility util;
-
- @BeforeClass
- public static void setupBeforeClass() throws Exception {
- util = new HBaseTestingUtility();
- util.startMiniCluster();
- createSchema();
- }
-
- private static void createSchema() throws IOException {
- new EntityTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- new AppToFlowTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- new ApplicationTable()
- .createTable(util.getHBaseAdmin(), util.getConfiguration());
- }
-
- @Test
- public void testWriteApplicationToHBase() throws Exception {
- TimelineEntities te = new TimelineEntities();
- ApplicationEntity entity = new ApplicationEntity();
- String id = "hello";
- entity.setId(id);
- long cTime = 1425016501000L;
- long mTime = 1425026901000L;
- entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
-
- // add the info map in Timeline Entity
- Map<String, Object> infoMap = new HashMap<String, Object>();
- infoMap.put("infoMapKey1", "infoMapValue1");
- infoMap.put("infoMapKey2", 10);
- entity.addInfo(infoMap);
-
- // add the isRelatedToEntity info
- String key = "task";
- String value = "is_related_to_entity_id_here";
- Set<String> isRelatedToSet = new HashSet<String>();
- isRelatedToSet.add(value);
- Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- isRelatedTo.put(key, isRelatedToSet);
- entity.setIsRelatedToEntities(isRelatedTo);
-
- // add the relatesTo info
- key = "container";
- value = "relates_to_entity_id_here";
- Set<String> relatesToSet = new HashSet<String>();
- relatesToSet.add(value);
- value = "relates_to_entity_id_here_Second";
- relatesToSet.add(value);
- Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- relatesTo.put(key, relatesToSet);
- entity.setRelatesToEntities(relatesTo);
-
- // add some config entries
- Map<String, String> conf = new HashMap<String, String>();
- conf.put("config_param1", "value1");
- conf.put("config_param2", "value2");
- entity.addConfigs(conf);
-
- // add metrics
- Set<TimelineMetric> metrics = new HashSet<>();
- TimelineMetric m1 = new TimelineMetric();
- m1.setId("MAP_SLOT_MILLIS");
- Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
- metricValues.put(ts - 120000, 100000000);
- metricValues.put(ts - 100000, 200000000);
- metricValues.put(ts - 80000, 300000000);
- metricValues.put(ts - 60000, 400000000);
- metricValues.put(ts - 40000, 50000000000L);
- metricValues.put(ts - 20000, 60000000000L);
- m1.setType(Type.TIME_SERIES);
- m1.setValues(metricValues);
- metrics.add(m1);
- entity.addMetrics(metrics);
-
- te.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_write_app";
- String user = "user1";
- String flow = "some_flow_name";
- String flowVersion = "AB7822C10F1111";
- long runid = 1002345678919L;
- hbi.write(cluster, user, flow, flowVersion, runid, id, te);
- hbi.stop();
-
- // retrieve the row
- byte[] rowKey =
- ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Connection conn = ConnectionFactory.createConnection(c1);
- Result result = new ApplicationTable().getResult(c1, conn, get);
-
- assertTrue(result != null);
- assertEquals(16, result.size());
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
- id));
-
- // check info column family
- String id1 = ApplicationColumn.ID.readResult(result).toString();
- assertEquals(id, id1);
-
- Number val =
- (Number) ApplicationColumn.CREATED_TIME.readResult(result);
- long cTime1 = val.longValue();
- assertEquals(cTime1, cTime);
-
- val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
- long mTime1 = val.longValue();
- assertEquals(mTime1, mTime);
-
- Map<String, Object> infoColumns =
- ApplicationColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap, infoColumns);
-
- // Remember isRelatedTo is of type Map<String, Set<String>>
- for (String isRelatedToKey : isRelatedTo.keySet()) {
- Object isRelatedToValue =
- ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
- isRelatedToKey);
- String compoundValue = isRelatedToValue.toString();
- // id7?id9?id6
- Set<String> isRelatedToValues =
- new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(isRelatedTo.get(isRelatedToKey).size(),
- isRelatedToValues.size());
- for (String v : isRelatedTo.get(isRelatedToKey)) {
- assertTrue(isRelatedToValues.contains(v));
- }
- }
-
- // RelatesTo
- for (String relatesToKey : relatesTo.keySet()) {
- String compoundValue =
- ApplicationColumnPrefix.RELATES_TO.readResult(result,
- relatesToKey).toString();
- // id3?id4?id5
- Set<String> relatesToValues =
- new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(relatesTo.get(relatesToKey).size(),
- relatesToValues.size());
- for (String v : relatesTo.get(relatesToKey)) {
- assertTrue(relatesToValues.contains(v));
- }
- }
-
- // Configuration
- Map<String, Object> configColumns =
- ApplicationColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf, configColumns);
-
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
-
- NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- assertEquals(metricValues, metricMap);
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- id, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertEquals(1, es1.size());
-
- // verify attributes
- assertEquals(id, e1.getId());
- assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
- e1.getType());
- assertEquals(cTime, e1.getCreatedTime());
- assertEquals(mTime, e1.getModifiedTime());
- Map<String, Object> infoMap2 = e1.getInfo();
- assertEquals(infoMap, infoMap2);
-
- Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
- assertEquals(isRelatedTo, isRelatedTo2);
-
- Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
- assertEquals(relatesTo, relatesTo2);
-
- Map<String, String> conf2 = e1.getConfigs();
- assertEquals(conf, conf2);
-
- Set<TimelineMetric> metrics2 = e1.getMetrics();
- assertEquals(metrics, metrics2);
- for (TimelineMetric metric2 : metrics2) {
- Map<Long, Number> metricValues2 = metric2.getValues();
- assertEquals(metricValues, metricValues2);
- }
- } finally {
- if (hbi != null) {
- hbi.stop();
- hbi.close();
- }
- if (hbr != null) {
- hbr.stop();
- hbr.close();
- }
- }
- }
-
- @Test
- public void testWriteEntityToHBase() throws Exception {
- TimelineEntities te = new TimelineEntities();
- TimelineEntity entity = new TimelineEntity();
- String id = "hello";
- String type = "world";
- entity.setId(id);
- entity.setType(type);
- long cTime = 1425016501000L;
- long mTime = 1425026901000L;
- entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
-
- // add the info map in Timeline Entity
- Map<String, Object> infoMap = new HashMap<String, Object>();
- infoMap.put("infoMapKey1", "infoMapValue1");
- infoMap.put("infoMapKey2", 10);
- entity.addInfo(infoMap);
-
- // add the isRelatedToEntity info
- String key = "task";
- String value = "is_related_to_entity_id_here";
- Set<String> isRelatedToSet = new HashSet<String>();
- isRelatedToSet.add(value);
- Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- isRelatedTo.put(key, isRelatedToSet);
- entity.setIsRelatedToEntities(isRelatedTo);
-
- // add the relatesTo info
- key = "container";
- value = "relates_to_entity_id_here";
- Set<String> relatesToSet = new HashSet<String>();
- relatesToSet.add(value);
- value = "relates_to_entity_id_here_Second";
- relatesToSet.add(value);
- Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- relatesTo.put(key, relatesToSet);
- entity.setRelatesToEntities(relatesTo);
-
- // add some config entries
- Map<String, String> conf = new HashMap<String, String>();
- conf.put("config_param1", "value1");
- conf.put("config_param2", "value2");
- entity.addConfigs(conf);
-
- // add metrics
- Set<TimelineMetric> metrics = new HashSet<>();
- TimelineMetric m1 = new TimelineMetric();
- m1.setId("MAP_SLOT_MILLIS");
- Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
- metricValues.put(ts - 120000, 100000000);
- metricValues.put(ts - 100000, 200000000);
- metricValues.put(ts - 80000, 300000000);
- metricValues.put(ts - 60000, 400000000);
- metricValues.put(ts - 40000, 50000000000L);
- metricValues.put(ts - 20000, 60000000000L);
- m1.setType(Type.TIME_SERIES);
- m1.setValues(metricValues);
- metrics.add(m1);
- entity.addMetrics(metrics);
-
- te.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_write_entity";
- String user = "user1";
- String flow = "some_flow_name";
- String flowVersion = "AB7822C10F1111";
- long runid = 1002345678919L;
- String appName = "some app name";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
- hbi.stop();
-
- // scan the table and see that entity exists
- Scan s = new Scan();
- byte[] startRow =
- EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
- s.setStartRow(startRow);
- s.setMaxVersions(Integer.MAX_VALUE);
- Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
-
- int rowCount = 0;
- int colCount = 0;
- for (Result result : scanner) {
- if (result != null && !result.isEmpty()) {
- rowCount++;
- colCount += result.size();
- byte[] row1 = result.getRow();
- assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
- entity));
-
- // check info column family
- String id1 = EntityColumn.ID.readResult(result).toString();
- assertEquals(id, id1);
-
- String type1 = EntityColumn.TYPE.readResult(result).toString();
- assertEquals(type, type1);
-
- Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
- long cTime1 = val.longValue();
- assertEquals(cTime1, cTime);
-
- val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
- long mTime1 = val.longValue();
- assertEquals(mTime1, mTime);
-
- Map<String, Object> infoColumns =
- EntityColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap, infoColumns);
-
- // Remember isRelatedTo is of type Map<String, Set<String>>
- for (String isRelatedToKey : isRelatedTo.keySet()) {
- Object isRelatedToValue =
- EntityColumnPrefix.IS_RELATED_TO.readResult(result,
- isRelatedToKey);
- String compoundValue = isRelatedToValue.toString();
- // id7?id9?id6
- Set<String> isRelatedToValues =
- new HashSet<String>(
- Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(isRelatedTo.get(isRelatedToKey).size(),
- isRelatedToValues.size());
- for (String v : isRelatedTo.get(isRelatedToKey)) {
- assertTrue(isRelatedToValues.contains(v));
- }
- }
-
- // RelatesTo
- for (String relatesToKey : relatesTo.keySet()) {
- String compoundValue =
- EntityColumnPrefix.RELATES_TO.readResult(result, relatesToKey)
- .toString();
- // id3?id4?id5
- Set<String> relatesToValues =
- new HashSet<String>(
- Separator.VALUES.splitEncoded(compoundValue));
- assertEquals(relatesTo.get(relatesToKey).size(),
- relatesToValues.size());
- for (String v : relatesTo.get(relatesToKey)) {
- assertTrue(relatesToValues.contains(v));
- }
- }
-
- // Configuration
- Map<String, Object> configColumns =
- EntityColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf, configColumns);
-
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
-
- NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- assertEquals(metricValues, metricMap);
- }
- }
- assertEquals(1, rowCount);
- assertEquals(17, colCount);
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertEquals(1, es1.size());
-
- // verify attributes
- assertEquals(id, e1.getId());
- assertEquals(type, e1.getType());
- assertEquals(cTime, e1.getCreatedTime());
- assertEquals(mTime, e1.getModifiedTime());
- Map<String, Object> infoMap2 = e1.getInfo();
- assertEquals(infoMap, infoMap2);
-
- Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
- assertEquals(isRelatedTo, isRelatedTo2);
-
- Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
- assertEquals(relatesTo, relatesTo2);
-
- Map<String, String> conf2 = e1.getConfigs();
- assertEquals(conf, conf2);
-
- Set<TimelineMetric> metrics2 = e1.getMetrics();
- assertEquals(metrics, metrics2);
- for (TimelineMetric metric2 : metrics2) {
- Map<Long, Number> metricValues2 = metric2.getValues();
- assertEquals(metricValues, metricValues2);
- }
- } finally {
- if (hbi != null) {
- hbi.stop();
- hbi.close();
- }
- if (hbr != null) {
- hbr.stop();
- hbr.close();
- }
- }
- }
-
- private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
- String flow, long runid, String appName, TimelineEntity te) {
-
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
-
- assertTrue(rowKeyComponents.length == 7);
- assertEquals(user, Bytes.toString(rowKeyComponents[0]));
- assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
- assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
- assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
- return true;
- }
-
- private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
- String user, String flow, long runid, String appName) {
-
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
-
- assertTrue(rowKeyComponents.length == 5);
- assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
- assertEquals(user, Bytes.toString(rowKeyComponents[1]));
- assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
- return true;
- }
-
- @Test
- public void testEvents() throws IOException {
- TimelineEvent event = new TimelineEvent();
- String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
- event.setId(eventId);
- long expTs = 1436512802000L;
- event.setTimestamp(expTs);
- String expKey = "foo_event";
- Object expVal = "test";
- event.addInfo(expKey, expVal);
-
- final TimelineEntity entity = new ApplicationEntity();
- entity.setId(ApplicationId.newInstance(0, 1).toString());
- entity.addEvent(event);
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_events";
- String user = "user2";
- String flow = "other_flow_name";
- String flowVersion = "1111F01C2287BA";
- long runid = 1009876543218L;
- String appName = "some app name";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
- hbi.stop();
-
- // retrieve the row
- byte[] rowKey =
- ApplicationRowKey.getRowKey(cluster, user, flow, runid, appName);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- Connection conn = ConnectionFactory.createConnection(c1);
- Result result = new ApplicationTable().getResult(c1, conn, get);
-
- assertTrue(result != null);
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
- appName));
-
- Map<?, Object> eventsResult =
- ApplicationColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- // there should be only one event
- assertEquals(1, eventsResult.size());
- for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
- // the qualifier is a compound key
- // hence match individual values
- byte[][] karr = (byte[][])e.getKey();
- assertEquals(3, karr.length);
- assertEquals(eventId, Bytes.toString(karr[0]));
- assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
- assertEquals(expKey, Bytes.toString(karr[2]));
- Object value = e.getValue();
- // there should be only one timestamp and value
- assertEquals(expVal, value.toString());
- }
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertNotNull(e2);
- assertEquals(e1, e2);
- assertEquals(1, es1.size());
- assertEquals(1, es2.size());
- assertEquals(es1, es2);
-
- // check the events
- NavigableSet<TimelineEvent> events = e1.getEvents();
- // there should be only one event
- assertEquals(1, events.size());
- for (TimelineEvent e : events) {
- assertEquals(eventId, e.getId());
- assertEquals(expTs, e.getTimestamp());
- Map<String,Object> info = e.getInfo();
- assertEquals(1, info.size());
- for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
- assertEquals(expKey, infoEntry.getKey());
- assertEquals(expVal, infoEntry.getValue());
- }
- }
- } finally {
- if (hbi != null) {
- hbi.stop();
- hbi.close();
- }
- if (hbr != null) {
- hbr.stop();
- hbr.close();
- }
- }
- }
-
- @Test
- public void testEventsWithEmptyInfo() throws IOException {
- TimelineEvent event = new TimelineEvent();
- String eventId = "foo_event_id";
- event.setId(eventId);
- long expTs = 1436512802000L;
- event.setTimestamp(expTs);
-
- final TimelineEntity entity = new TimelineEntity();
- entity.setId("attempt_1329348432655_0001_m_000008_18");
- entity.setType("FOO_ATTEMPT");
- entity.addEvent(event);
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
-
- HBaseTimelineWriterImpl hbi = null;
- HBaseTimelineReaderImpl hbr = null;
- try {
- Configuration c1 = util.getConfiguration();
- hbi = new HBaseTimelineWriterImpl(c1);
- hbi.init(c1);
- hbi.start();
- hbr = new HBaseTimelineReaderImpl();
- hbr.init(c1);
- hbr.start();
- String cluster = "cluster_test_empty_eventkey";
- String user = "user_emptyeventkey";
- String flow = "other_flow_name";
- String flowVersion = "1111F01C2287BA";
- long runid = 1009876543218L;
- String appName = "some app name";
- byte[] startRow =
- EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
- hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
- hbi.stop();
- // scan the table and see that entity exists
- Scan s = new Scan();
- s.setStartRow(startRow);
- s.addFamily(EntityColumnFamily.INFO.getBytes());
- Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
-
- int rowCount = 0;
- for (Result result : scanner) {
- if (result != null && !result.isEmpty()) {
- rowCount++;
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
- entity));
-
- Map<?, Object> eventsResult =
- EntityColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- // there should be only one event
- assertEquals(1, eventsResult.size());
- for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
- // the qualifier is a compound key
- // hence match individual values
- byte[][] karr = (byte[][])e.getKey();
- assertEquals(3, karr.length);
- assertEquals(eventId, Bytes.toString(karr[0]));
- assertEquals(TimelineWriterUtils.invert(expTs),
- Bytes.toLong(karr[1]));
- // key must be empty
- assertEquals(0, karr[2].length);
- Object value = e.getValue();
- // value should be empty
- assertEquals("", value.toString());
- }
- }
- }
- assertEquals(1, rowCount);
-
- // read the timeline entity using the reader this time
- TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(),
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
- assertNotNull(e1);
- assertEquals(1, es1.size());
-
- // check the events
- NavigableSet<TimelineEvent> events = e1.getEvents();
- // there should be only one event
- assertEquals(1, events.size());
- for (TimelineEvent e : events) {
- assertEquals(eventId, e.getId());
- assertEquals(expTs, e.getTimestamp());
- Map<String,Object> info = e.getInfo();
- assertTrue(info == null || info.isEmpty());
- }
- } finally {
- hbi.stop();
- hbi.close();
- hbr.stop();;
- hbr.close();
- }
- }
-
- @AfterClass
- public static void tearDownAfterClass() throws Exception {
- util.shutdownMiniCluster();
- }
-}
[49/50] [abbrv] hadoop git commit: YARN-4025. Deal with byte
representations of Longs in writer code. Contributed by Sangjin Lee and
Vrushali C.
Posted by sj...@apache.org.
YARN-4025. Deal with byte representations of Longs in writer code. Contributed by Sangjin Lee and Vrushali C.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89f8fd2e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89f8fd2e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89f8fd2e
Branch: refs/heads/YARN-2928
Commit: 89f8fd2ebc97b9f96e329fdf1ed0692f12d3faee
Parents: 702a214
Author: Junping Du <ju...@apache.org>
Authored: Wed Aug 19 10:00:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:52:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/HBaseTimelineReaderImpl.java | 68 +++---
.../storage/HBaseTimelineWriterImpl.java | 20 +-
.../application/ApplicationColumnPrefix.java | 40 ++++
.../storage/application/ApplicationTable.java | 6 +-
.../storage/common/ColumnHelper.java | 99 ++++++++-
.../storage/common/Separator.java | 16 +-
.../storage/common/TimelineWriterUtils.java | 9 +-
.../storage/entity/EntityColumnPrefix.java | 40 ++++
.../storage/entity/EntityTable.java | 6 +-
.../storage/TestHBaseTimelineWriterImpl.java | 207 +++++++++++--------
11 files changed, 373 insertions(+), 141 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 492a098..cd7f849 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -94,6 +94,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3045. Implement NM writing container lifecycle events to Timeline
Service v2. (Naganarasimha G R via junping_du)
+ YARN-4025. Deal with byte representations of Longs in writer code.
+ (Sangjin Lee and Vrushali C via junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 094f868..c514c20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -19,12 +19,9 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NavigableSet;
@@ -431,44 +428,51 @@ public class HBaseTimelineReaderImpl
Map<String, Object> columns = prefix.readResults(result);
if (isConfig) {
for (Map.Entry<String, Object> column : columns.entrySet()) {
- entity.addConfig(column.getKey(), column.getKey().toString());
+ entity.addConfig(column.getKey(), column.getValue().toString());
}
} else {
entity.addInfo(columns);
}
}
+ /**
+ * Read events from the entity table or the application table. The column name
+ * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
+ * if there is no info associated with the event.
+ *
+ * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
+ * schema description.
+ */
private static void readEvents(TimelineEntity entity, Result result,
boolean isApplication) throws IOException {
Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<String, Object> eventsResult = isApplication ?
- ApplicationColumnPrefix.EVENT.readResults(result) :
- EntityColumnPrefix.EVENT.readResults(result);
- for (Map.Entry<String,Object> eventResult : eventsResult.entrySet()) {
- Collection<String> tokens =
- Separator.VALUES.splitEncoded(eventResult.getKey());
- if (tokens.size() != 2 && tokens.size() != 3) {
- throw new IOException(
- "Invalid event column name: " + eventResult.getKey());
- }
- Iterator<String> idItr = tokens.iterator();
- String id = idItr.next();
- String tsStr = idItr.next();
- // TODO: timestamp is not correct via ser/des through UTF-8 string
- Long ts =
- TimelineWriterUtils.invert(Bytes.toLong(tsStr.getBytes(
- StandardCharsets.UTF_8)));
- String key = Separator.VALUES.joinEncoded(id, ts.toString());
- TimelineEvent event = eventsMap.get(key);
- if (event == null) {
- event = new TimelineEvent();
- event.setId(id);
- event.setTimestamp(ts);
- eventsMap.put(key, event);
- }
- if (tokens.size() == 3) {
- String infoKey = idItr.next();
- event.addInfo(infoKey, eventResult.getValue());
+ Map<?, Object> eventsResult = isApplication ?
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result) :
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
+ byte[][] karr = (byte[][])eventResult.getKey();
+ // the column name is of the form "eventId=timestamp=infoKey"
+ if (karr.length == 3) {
+ String id = Bytes.toString(karr[0]);
+ long ts = TimelineWriterUtils.invert(Bytes.toLong(karr[1]));
+ String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ // handle empty info
+ String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
+ if (infoKey != null) {
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ } else {
+ LOG.warn("incorrectly formatted column name: it will be discarded");
+ continue;
}
}
Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 96192cc..772002d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -300,25 +300,27 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
byte[] compoundColumnQualifierBytes =
Separator.VALUES.join(columnQualifierWithTsBytes,
null);
- String compoundColumnQualifier =
- Bytes.toString(compoundColumnQualifierBytes);
- EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifier, null, TimelineWriterUtils.EMPTY_BYTES);
+ if (isApplication) {
+ ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
+ compoundColumnQualifierBytes, null,
+ TimelineWriterUtils.EMPTY_BYTES);
+ } else {
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifierBytes, null,
+ TimelineWriterUtils.EMPTY_BYTES);
+ }
} else {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
// eventId?infoKey
byte[] compoundColumnQualifierBytes =
Separator.VALUES.join(columnQualifierWithTsBytes,
Bytes.toBytes(info.getKey()));
- // convert back to string to avoid additional API on store.
- String compoundColumnQualifier =
- Bytes.toString(compoundColumnQualifierBytes);
if (isApplication) {
ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
- compoundColumnQualifier, null, info.getValue());
+ compoundColumnQualifierBytes, null, info.getValue());
} else {
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifier, null, info.getValue());
+ compoundColumnQualifierBytes, null, info.getValue());
}
} // for info: eventInfo
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
index cd9e845..ad1def6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -111,6 +111,31 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
* TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
*/
public void store(byte[] rowKey,
+ TypedBufferedMutator<ApplicationTable> tableMutator, byte[] qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
TypedBufferedMutator<ApplicationTable> tableMutator, String qualifier,
Long timestamp, Object inputValue) throws IOException {
@@ -150,6 +175,21 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
return column.readResults(result, columnPrefixBytes);
}
+ /**
+ * @param result from which to read columns
+ * @return the latest values of columns in the column family. The column
+ * qualifier is returned as a list of parts, each part a byte[]. This
+ * is to facilitate returning byte arrays of values that were not
+ * Strings. If they can be treated as Strings, you should use
+ * {@link #readResults(Result)} instead.
+ * @throws IOException
+ */
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException {
+ return column.readResultsHavingCompoundColumnQualifiers(result,
+ columnPrefixBytes);
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index d2a2cb9..a997997 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -57,12 +57,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | | infoValue | metricValue1 | |
* | | | @timestamp2 | |
* | | r!relatesToKey: | | |
- * | | id3?id4?id5 | | |
+ * | | id3=id4=id5 | | |
* | | | | |
* | | s!isRelatedToKey: | | |
- * | | id7?id9?id6 | | |
+ * | | id7=id9=id6 | | |
* | | | | |
- * | | e!eventId?timestamp?infoKey: | | |
+ * | | e!eventId=timestamp=infoKey: | | |
* | | eventInfoValue | | |
* | | | | |
* | | flowVersion: | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index a902924..f1b7c58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -24,6 +24,8 @@ import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.TreeMap;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
@@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
* @param <T> refers to the table.
*/
public class ColumnHelper<T> {
+ private static final Log LOG = LogFactory.getLog(ColumnHelper.class);
private final ColumnFamily<T> columnFamily;
@@ -143,6 +146,7 @@ public class ColumnHelper<T> {
.entrySet()) {
String columnName = null;
if (columnPrefixBytes == null) {
+ LOG.info("null prefix was specified; returning all columns");
// Decode the spaces we encoded in the column name.
columnName = Separator.decode(entry.getKey(), Separator.SPACE);
} else {
@@ -181,32 +185,43 @@ public class ColumnHelper<T> {
/**
* @param result from which to read columns
* @param columnPrefixBytes optional prefix to limit columns. If null all
- * columns are returned.
- * @return the latest values of columns in the column family.
+ * columns are returned.
+ * @return the latest values of columns in the column family. This assumes
+ * that the column name parts are all Strings by default. If the
+ * column name parts should be treated natively and not be converted
+ * back and forth from Strings, you should use
+ * {@link #readResultsHavingCompoundColumnQualifiers(Result, byte[])}
+ * instead.
* @throws IOException
*/
- public Map<String, Object> readResults(Result result, byte[] columnPrefixBytes)
- throws IOException {
+ public Map<String, Object> readResults(Result result,
+ byte[] columnPrefixBytes) throws IOException {
Map<String, Object> results = new HashMap<String, Object>();
if (result != null) {
Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
for (Entry<byte[], byte[]> entry : columns.entrySet()) {
- if (entry.getKey() != null && entry.getKey().length > 0) {
+ byte[] columnKey = entry.getKey();
+ if (columnKey != null && columnKey.length > 0) {
String columnName = null;
if (columnPrefixBytes == null) {
+ LOG.info("null prefix was specified; returning all columns");
// Decode the spaces we encoded in the column name.
- columnName = Separator.decode(entry.getKey(), Separator.SPACE);
+ columnName = Separator.decode(columnKey, Separator.SPACE);
} else {
// A non-null prefix means columns are actually of the form
// prefix!columnNameRemainder
byte[][] columnNameParts =
- Separator.QUALIFIERS.split(entry.getKey(), 2);
+ Separator.QUALIFIERS.split(columnKey, 2);
byte[] actualColumnPrefixBytes = columnNameParts[0];
if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
&& columnNameParts.length == 2) {
// This is the prefix that we want
+ // if the column name is a compound qualifier
+ // with non string datatypes, the following decode will not
+ // work correctly since it considers all components to be String
+ // invoke the readResultsHavingCompoundColumnQualifiers function
columnName = Separator.decode(columnNameParts[1]);
}
}
@@ -223,6 +238,56 @@ public class ColumnHelper<T> {
}
/**
+ * @param result from which to read columns
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @return the latest values of columns in the column family. If the column
+ * prefix is null, the column qualifier is returned as Strings. For a
+ * non-null column prefix bytes, the column qualifier is returned as
+ * a list of parts, each part a byte[]. This is to facilitate
+ * returning byte arrays of values that were not Strings.
+ * @throws IOException
+ */
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result,
+ byte[] columnPrefixBytes) throws IOException {
+ // handle the case where the column prefix is null
+ // it is the same as readResults() so simply delegate to that implementation
+ if (columnPrefixBytes == null) {
+ return readResults(result, null);
+ }
+
+ Map<byte[][], Object> results = new HashMap<byte[][], Object>();
+
+ if (result != null) {
+ Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
+ for (Entry<byte[], byte[]> entry : columns.entrySet()) {
+ byte[] columnKey = entry.getKey();
+ if (columnKey != null && columnKey.length > 0) {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ // with a compound column qualifier, we are presuming existence of a
+ // prefix
+ byte[][] columnNameParts = Separator.QUALIFIERS.split(columnKey, 2);
+ if (columnNameParts.length > 0) {
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ // This is the prefix that we want
+ byte[][] columnQualifierParts =
+ Separator.VALUES.split(columnNameParts[1]);
+ Object value = GenericObjectMapper.read(entry.getValue());
+ // we return the columnQualifier in parts since we don't know
+ // which part is of which data type
+ results.put(columnQualifierParts, value);
+ }
+ }
+ }
+ } // for entry
+ }
+ return results;
+ }
+
+ /**
* @param columnPrefixBytes The byte representation for the column prefix.
* Should not contain {@link Separator#QUALIFIERS}.
* @param qualifier for the remainder of the column. Any
@@ -247,4 +312,24 @@ public class ColumnHelper<T> {
return columnQualifier;
}
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier the byte representation for the remainder of the column.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ byte[] qualifier) {
+
+ if (columnPrefixBytes == null) {
+ return qualifier;
+ }
+
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, qualifier);
+ return columnQualifier;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
index 3319419..9f91af8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -37,7 +37,7 @@ public enum Separator {
/**
* separator in values, and/or compound key/column qualifier fields.
*/
- VALUES("?", "%1$"),
+ VALUES("=", "%1$"),
/**
* separator in values, often used to avoid having these in qualifiers and
@@ -299,12 +299,22 @@ public enum Separator {
* up to a maximum of count items. This will naturally produce copied byte
* arrays for each of the split segments.
* @param source to be split
- * @param limit on how many segments are supposed to be returned. Negative
- * value indicates no limit on number of segments.
+ * @param limit on how many segments are supposed to be returned. A
+ * non-positive value indicates no limit on number of segments.
* @return source split by this separator.
*/
public byte[][] split(byte[] source, int limit) {
return TimelineWriterUtils.split(source, this.bytes, limit);
}
+ /**
+ * Splits the source array into multiple array segments using this separator,
+ * as many times as splits are found. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ * @param source to be split
+ * @return source split by this separator.
+ */
+ public byte[][] split(byte[] source) {
+ return TimelineWriterUtils.split(source, this.bytes);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
index c957bf5..58bdedc7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -33,6 +33,9 @@ public class TimelineWriterUtils {
/** empty bytes */
public static final byte[] EMPTY_BYTES = new byte[0];
+ /** indicator for no limits for splitting */
+ public static final int NO_LIMIT_SPLIT = -1;
+
/**
* Splits the source array into multiple array segments using the given
* separator, up to a maximum of count items. This will naturally produce
@@ -45,7 +48,7 @@ public class TimelineWriterUtils {
* @return byte[] array after splitting the source
*/
public static byte[][] split(byte[] source, byte[] separator) {
- return split(source, separator, -1);
+ return split(source, separator, NO_LIMIT_SPLIT);
}
/**
@@ -57,7 +60,7 @@ public class TimelineWriterUtils {
*
* @param source
* @param separator
- * @param limit a negative value indicates no limit on number of segments.
+ * @param limit a non-positive value indicates no limit on number of segments.
* @return byte[][] after splitting the input source
*/
public static byte[][] split(byte[] source, byte[] separator, int limit) {
@@ -81,7 +84,7 @@ public class TimelineWriterUtils {
* separator byte array.
*/
public static List<Range> splitRanges(byte[] source, byte[] separator) {
- return splitRanges(source, separator, -1);
+ return splitRanges(source, separator, NO_LIMIT_SPLIT);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index c8485c0..75ff742 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -131,6 +131,31 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*
* @see
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<EntityTable> tableMutator, byte[] qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
*/
public Object readResult(Result result, String qualifier) throws IOException {
@@ -150,6 +175,21 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
return column.readResults(result, columnPrefixBytes);
}
+ /**
+ * @param result from which to read columns
+ * @return the latest values of columns in the column family. The column
+ * qualifier is returned as a list of parts, each part a byte[]. This
+ * is to facilitate returning byte arrays of values that were not
+ * Strings. If they can be treated as Strings, you should use
+ * {@link #readResults(Result)} instead.
+ * @throws IOException
+ */
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException {
+ return column.readResultsHavingCompoundColumnQualifiers(result,
+ columnPrefixBytes);
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 9a8bd8c..96773b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -58,12 +58,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | | infoValue | | |
* | | | | |
* | | r!relatesToKey: | | |
- * | | id3?id4?id5 | | |
+ * | | id3=id4=id5 | | |
* | | | | |
* | | s!isRelatedToKey | | |
- * | | id7?id9?id6 | | |
+ * | | id7=id9=id6 | | |
* | | | | |
- * | | e!eventId?timestamp?infoKey: | | |
+ * | | e!eventId=timestamp=infoKey: | | |
* | | eventInfoValue | | |
* | | | | |
* | | flowVersion: | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89f8fd2e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 95f88d1..2875e01 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -27,8 +27,8 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.NavigableMap;
+import java.util.NavigableSet;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
@@ -75,7 +76,7 @@ import org.junit.Test;
* even if other records exist in the table. Use a different cluster name if
* you add a new test.
*/
-public class TestHBaseTimelineWriterImpl {
+public class TestHBaseTimelineStorage {
private static HBaseTestingUtility util;
@@ -101,8 +102,8 @@ public class TestHBaseTimelineWriterImpl {
ApplicationEntity entity = new ApplicationEntity();
String id = "hello";
entity.setId(id);
- Long cTime = 1425016501000L;
- Long mTime = 1425026901000L;
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
@@ -197,19 +198,16 @@ public class TestHBaseTimelineWriterImpl {
Number val =
(Number) ApplicationColumn.CREATED_TIME.readResult(result);
- Long cTime1 = val.longValue();
+ long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
- Long mTime1 = val.longValue();
+ long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
Map<String, Object> infoColumns =
ApplicationColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap.size(), infoColumns.size());
- for (String infoItem : infoMap.keySet()) {
- assertEquals(infoMap.get(infoItem), infoColumns.get(infoItem));
- }
+ assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (String isRelatedToKey : isRelatedTo.keySet()) {
@@ -245,27 +243,15 @@ public class TestHBaseTimelineWriterImpl {
// Configuration
Map<String, Object> configColumns =
ApplicationColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf.size(), configColumns.size());
- for (String configItem : conf.keySet()) {
- assertEquals(conf.get(configItem), configColumns.get(configItem));
- }
+ assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- // We got metrics back
- assertNotNull(metricMap);
- // Same number of metrics as we wrote
- assertEquals(metricValues.entrySet().size(), metricMap.entrySet().size());
-
- // Iterate over original metrics and confirm that they are present
- // here.
- for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
- assertEquals(metricEntry.getValue(),
- metricMap.get(metricEntry.getKey()));
- }
+ assertEquals(metricValues, metricMap);
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -274,6 +260,31 @@ public class TestHBaseTimelineWriterImpl {
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
+ e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
} finally {
if (hbi != null) {
hbi.stop();
@@ -294,8 +305,8 @@ public class TestHBaseTimelineWriterImpl {
String type = "world";
entity.setId(id);
entity.setType(type);
- Long cTime = 1425016501000L;
- Long mTime = 1425026901000L;
+ long cTime = 1425016501000L;
+ long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
entity.setModifiedTime(mTime);
@@ -396,20 +407,16 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(type, type1);
Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
- Long cTime1 = val.longValue();
+ long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
- Long mTime1 = val.longValue();
+ long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
Map<String, Object> infoColumns =
EntityColumnPrefix.INFO.readResults(result);
- assertEquals(infoMap.size(), infoColumns.size());
- for (String infoItem : infoMap.keySet()) {
- assertEquals(infoMap.get(infoItem),
- infoColumns.get(infoItem));
- }
+ assertEquals(infoMap, infoColumns);
// Remember isRelatedTo is of type Map<String, Set<String>>
for (String isRelatedToKey : isRelatedTo.keySet()) {
@@ -447,32 +454,19 @@ public class TestHBaseTimelineWriterImpl {
// Configuration
Map<String, Object> configColumns =
EntityColumnPrefix.CONFIG.readResults(result);
- assertEquals(conf.size(), configColumns.size());
- for (String configItem : conf.keySet()) {
- assertEquals(conf.get(configItem), configColumns.get(configItem));
- }
+ assertEquals(conf, configColumns);
NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
- // We got metrics back
- assertNotNull(metricMap);
- // Same number of metrics as we wrote
- assertEquals(metricValues.entrySet().size(), metricMap.entrySet()
- .size());
-
- // Iterate over original metrics and confirm that they are present
- // here.
- for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
- assertEquals(metricEntry.getValue(),
- metricMap.get(metricEntry.getKey()));
- }
+ assertEquals(metricValues, metricMap);
}
}
assertEquals(1, rowCount);
assertEquals(17, colCount);
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -481,6 +475,30 @@ public class TestHBaseTimelineWriterImpl {
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
+
+ // verify attributes
+ assertEquals(id, e1.getId());
+ assertEquals(type, e1.getType());
+ assertEquals(cTime, e1.getCreatedTime());
+ assertEquals(mTime, e1.getModifiedTime());
+ Map<String, Object> infoMap2 = e1.getInfo();
+ assertEquals(infoMap, infoMap2);
+
+ Map<String, Set<String>> isRelatedTo2 = e1.getIsRelatedToEntities();
+ assertEquals(isRelatedTo, isRelatedTo2);
+
+ Map<String, Set<String>> relatesTo2 = e1.getRelatesToEntities();
+ assertEquals(relatesTo, relatesTo2);
+
+ Map<String, String> conf2 = e1.getConfigs();
+ assertEquals(conf, conf2);
+
+ Set<TimelineMetric> metrics2 = e1.getMetrics();
+ assertEquals(metrics, metrics2);
+ for (TimelineMetric metric2 : metrics2) {
+ Map<Long, Number> metricValues2 = metric2.getValues();
+ assertEquals(metricValues, metricValues2);
+ }
} finally {
if (hbi != null) {
hbi.stop();
@@ -494,9 +512,9 @@ public class TestHBaseTimelineWriterImpl {
}
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
- String flow, Long runid, String appName, TimelineEntity te) {
+ String flow, long runid, String appName, TimelineEntity te) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
assertTrue(rowKeyComponents.length == 7);
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
@@ -511,9 +529,9 @@ public class TestHBaseTimelineWriterImpl {
}
private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
- String user, String flow, Long runid, String appName) {
+ String user, String flow, long runid, String appName) {
- byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
assertTrue(rowKeyComponents.length == 5);
assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
@@ -530,7 +548,7 @@ public class TestHBaseTimelineWriterImpl {
TimelineEvent event = new TimelineEvent();
String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
event.setId(eventId);
- Long expTs = 1436512802000L;
+ long expTs = 1436512802000L;
event.setTimestamp(expTs);
String expKey = "foo_event";
Object expVal = "test";
@@ -577,24 +595,25 @@ public class TestHBaseTimelineWriterImpl {
assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
appName));
- Map<String, Object> eventsResult =
- ApplicationColumnPrefix.EVENT.readResults(result);
+ Map<?, Object> eventsResult =
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
// there should be only one event
assertEquals(1, eventsResult.size());
- // key name for the event
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(Bytes.toBytes(eventId),
- Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
- Bytes.toBytes(expKey));
- String valueKey = Bytes.toString(compoundColumnQualifierBytes);
- for (Map.Entry<String, Object> e : eventsResult.entrySet()) {
- // the value key must match
- assertEquals(valueKey, e.getKey());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs), Bytes.toLong(karr[1]));
+ assertEquals(expKey, Bytes.toString(karr[2]));
Object value = e.getValue();
// there should be only one timestamp and value
assertEquals(expVal, value.toString());
}
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -613,6 +632,21 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(1, es1.size());
assertEquals(1, es2.size());
assertEquals(es1, es2);
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertEquals(1, info.size());
+ for (Map.Entry<String, Object> infoEntry : info.entrySet()) {
+ assertEquals(expKey, infoEntry.getKey());
+ assertEquals(expVal, infoEntry.getValue());
+ }
+ }
} finally {
if (hbi != null) {
hbi.stop();
@@ -630,7 +664,7 @@ public class TestHBaseTimelineWriterImpl {
TimelineEvent event = new TimelineEvent();
String eventId = "foo_event_id";
event.setId(eventId);
- Long expTs = 1436512802000L;
+ long expTs = 1436512802000L;
event.setTimestamp(expTs);
final TimelineEntity entity = new TimelineEntity();
@@ -678,22 +712,21 @@ public class TestHBaseTimelineWriterImpl {
assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
entity));
- Map<String, Object> eventsResult =
- EntityColumnPrefix.EVENT.readResults(result);
+ Map<?, Object> eventsResult =
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
// there should be only one event
assertEquals(1, eventsResult.size());
- // key name for the event
- byte[] compoundColumnQualifierWithTsBytes =
- Separator.VALUES.join(Bytes.toBytes(eventId),
- Bytes.toBytes(TimelineWriterUtils.invert(expTs)));
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(compoundColumnQualifierWithTsBytes,
- null);
- String valueKey = Bytes.toString(compoundColumnQualifierBytes);
- for (Map.Entry<String, Object> e :
- eventsResult.entrySet()) {
- // the column qualifier key must match
- assertEquals(valueKey, e.getKey());
+ for (Map.Entry<?, Object> e : eventsResult.entrySet()) {
+ // the qualifier is a compound key
+ // hence match individual values
+ byte[][] karr = (byte[][])e.getKey();
+ assertEquals(3, karr.length);
+ assertEquals(eventId, Bytes.toString(karr[0]));
+ assertEquals(TimelineWriterUtils.invert(expTs),
+ Bytes.toLong(karr[1]));
+ // key must be empty
+ assertEquals(0, karr[2].length);
Object value = e.getValue();
// value should be empty
assertEquals("", value.toString());
@@ -702,6 +735,7 @@ public class TestHBaseTimelineWriterImpl {
}
assertEquals(1, rowCount);
+ // read the timeline entity using the reader this time
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
entity.getType(), entity.getId(),
EnumSet.of(TimelineReader.Field.ALL));
@@ -710,6 +744,17 @@ public class TestHBaseTimelineWriterImpl {
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
+
+ // check the events
+ NavigableSet<TimelineEvent> events = e1.getEvents();
+ // there should be only one event
+ assertEquals(1, events.size());
+ for (TimelineEvent e : events) {
+ assertEquals(eventId, e.getId());
+ assertEquals(expTs, e.getTimestamp());
+ Map<String,Object> info = e.getInfo();
+ assertTrue(info == null || info.isEmpty());
+ }
} finally {
hbi.stop();
hbi.close();
[35/50] [abbrv] hadoop git commit: YARN-3047. [Data Serving] Set up
ATS reader with basic request serving structure and lifecycle (Varun Saxena
via sjlee)
Posted by sj...@apache.org.
YARN-3047. [Data Serving] Set up ATS reader with basic request serving structure and lifecycle (Varun Saxena via sjlee)
(cherry picked from commit 4c5f88fb0f04b7919738d07598b0f006a9ff91f2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e281987f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e281987f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e281987f
Branch: refs/heads/YARN-2928
Commit: e281987f7b78295c8f511b6c63ae8d8b2a7cc8a9
Parents: 89e6c69
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Jul 8 17:10:10 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:14 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +
hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd | 8 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 3 +
.../hadoop/yarn/webapp/util/WebAppUtils.java | 6 +-
.../reader/TimelineReaderManager.java | 36 ++++
.../reader/TimelineReaderServer.java | 169 +++++++++++++++++++
.../reader/TimelineReaderWebServices.java | 59 +++++++
.../reader/TestTimelineReaderServer.java | 51 ++++++
.../reader/TestTimelineReaderWebServices.java | 121 +++++++++++++
10 files changed, 459 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 58232b7..d77ad59 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -76,6 +76,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3051. Created storage oriented reader interface for fetching raw entity
data and made the filesystem based implementation. (Varun Saxena via zjshen)
+ YARN-3047. [Data Serving] Set up ATS reader with basic request serving
+ structure and lifecycle (Varun Saxena via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index f0bed9b..b6d1204 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -43,6 +43,7 @@ function hadoop_usage
hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools"
hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon"
hadoop_add_subcommand "timelineserver" "run the timeline server"
+ hadoop_add_subcommand "timelinereader" "run the timeline reader server"
hadoop_add_subcommand "top" "view cluster information"
hadoop_add_subcommand "version" "print the version"
hadoop_generate_usage "${MYNAME}" true
@@ -179,6 +180,10 @@ case "${COMMAND}" in
HADOOP_HEAPSIZE_MAX="${YARN_TIMELINESERVER_HEAPSIZE}"
fi
;;
+ timelinereader)
+ supportdaemonization="true"
+ CLASS='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
+ ;;
version)
CLASS=org.apache.hadoop.util.VersionInfo
hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
index 91c90fb..4c36307 100644
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn.cmd
@@ -151,7 +151,7 @@ if "%1" == "--loglevel" (
set yarncommands=resourcemanager nodemanager proxyserver rmadmin version jar ^
application applicationattempt container node queue logs daemonlog historyserver ^
- timelineserver classpath
+ timelineserver timelinereader classpath
for %%i in ( %yarncommands% ) do (
if %yarn-command% == %%i set yarncommand=true
)
@@ -242,6 +242,11 @@ goto :eof
)
goto :eof
+:timelinereader
+ set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\timelineserver-config\log4j.properties
+ set CLASS=org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer
+ goto :eof
+
:nodemanager
set CLASSPATH=%CLASSPATH%;%YARN_CONF_DIR%\nm-config\log4j.properties
set CLASS=org.apache.hadoop.yarn.server.nodemanager.NodeManager
@@ -312,6 +317,7 @@ goto :eof
@echo resourcemanager run the ResourceManager
@echo nodemanager run a nodemanager on each slave
@echo timelineserver run the timeline server
+ @echo timelinereader run the timeline reader server
@echo rmadmin admin tools
@echo version print the version
@echo jar ^<jar^> run a jar file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8640864..2bc7d70 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1467,6 +1467,9 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_WRITER_CLASS =
TIMELINE_SERVICE_PREFIX + "writer.class";
+ public static final String TIMELINE_SERVICE_READER_CLASS =
+ TIMELINE_SERVICE_PREFIX + "reader.class";
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
index 8597427..f4a7416 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
@@ -263,6 +263,10 @@ public class WebAppUtils {
}
public static String getAHSWebAppURLWithoutScheme(Configuration conf) {
+ return getTimelineReaderWebAppURL(conf);
+ }
+
+ public static String getTimelineReaderWebAppURL(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS);
@@ -271,7 +275,7 @@ public class WebAppUtils {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS);
}
}
-
+
/**
* if url has scheme then it will be returned as it is else it will return
* url with scheme.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
new file mode 100644
index 0000000..5573185
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+
+@Private
+@Unstable
+public class TimelineReaderManager extends AbstractService {
+
+ private TimelineReader reader;
+
+ public TimelineReaderManager(TimelineReader timelineReader) {
+ super(TimelineReaderManager.class.getName());
+ this.reader = timelineReader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
new file mode 100644
index 0000000..874112c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -0,0 +1,169 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.util.ExitUtil;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/** Main class for Timeline Reader */
+@Private
+@Unstable
+public class TimelineReaderServer extends CompositeService {
+ private static final Log LOG = LogFactory.getLog(TimelineReaderServer.class);
+ private static final int SHUTDOWN_HOOK_PRIORITY = 30;
+ private static final String TIMELINE_READER_MANAGER_ATTR =
+ "timeline.reader.manager";
+
+ private HttpServer2 readerWebServer;
+ private TimelineReaderManager timelineReaderManager;
+
+ public TimelineReaderServer() {
+ super(TimelineReaderServer.class.getName());
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ TimelineReader timelineReaderStore = createTimelineReaderStore(conf);
+ addService(timelineReaderStore);
+ timelineReaderManager = createTimelineReaderManager(timelineReaderStore);
+ addService(timelineReaderManager);
+ super.serviceInit(conf);
+ }
+
+ private TimelineReader createTimelineReaderStore(Configuration conf) {
+ TimelineReader readerStore = ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.TIMELINE_SERVICE_READER_CLASS,
+ FileSystemTimelineReaderImpl.class, TimelineReader.class), conf);
+ readerStore.init(conf);
+ return readerStore;
+ }
+
+ private TimelineReaderManager createTimelineReaderManager(
+ TimelineReader timelineReaderStore) {
+ return new TimelineReaderManager(timelineReaderStore);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ startTimelineReaderWebApp();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (readerWebServer != null) {
+ readerWebServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ private void startTimelineReaderWebApp() {
+ Configuration conf = getConfig();
+ String bindAddress = WebAppUtils.getWebAppBindURL(conf,
+ YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ WebAppUtils.getTimelineReaderWebAppURL(conf));
+ LOG.info("Instantiating TimelineReaderWebApp at " + bindAddress);
+ try {
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("timeline")
+ .setConf(conf)
+ .addEndpoint(URI.create("http://" + bindAddress));
+ readerWebServer = builder.build();
+
+ Map<String, String> options = new HashMap<>();
+ String username = conf.get(HADOOP_HTTP_STATIC_USER,
+ DEFAULT_HADOOP_HTTP_STATIC_USER);
+ options.put(HADOOP_HTTP_STATIC_USER, username);
+ HttpServer2.defineFilter(readerWebServer.getWebAppContext(),
+ "static_user_filter_timeline",
+ StaticUserWebFilter.StaticUserFilter.class.getName(),
+ options, new String[] {"/*"});
+
+ readerWebServer.addJerseyResourcePackage(
+ TimelineReaderWebServices.class.getPackage().getName() + ";"
+ + GenericExceptionHandler.class.getPackage().getName() + ";"
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
+ "/*");
+ readerWebServer.setAttribute(TIMELINE_READER_MANAGER_ATTR,
+ timelineReaderManager);
+ readerWebServer.start();
+ } catch (Exception e) {
+ String msg = "TimelineReaderWebApp failed to start.";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ }
+
+ @VisibleForTesting
+ int getWebServerPort() {
+ return readerWebServer.getConnectorAddress(0).getPort();
+ }
+
+ static TimelineReaderServer startTimelineReaderServer(String[] args) {
+ Thread.setDefaultUncaughtExceptionHandler(
+ new YarnUncaughtExceptionHandler());
+ StringUtils.startupShutdownMessage(TimelineReaderServer.class,
+ args, LOG);
+ TimelineReaderServer timelineReaderServer = null;
+ try {
+ timelineReaderServer = new TimelineReaderServer();
+ ShutdownHookManager.get().addShutdownHook(
+ new CompositeServiceShutdownHook(timelineReaderServer),
+ SHUTDOWN_HOOK_PRIORITY);
+ YarnConfiguration conf = new YarnConfiguration();
+ timelineReaderServer.init(conf);
+ timelineReaderServer.start();
+ } catch (Throwable t) {
+ LOG.fatal("Error starting TimelineReaderWebServer", t);
+ ExitUtil.terminate(-1, "Error starting TimelineReaderWebServer");
+ }
+ return timelineReaderServer;
+ }
+
+ public static void main(String[] args) {
+ startTimelineReaderServer(args);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
new file mode 100644
index 0000000..3655a72
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.Produces;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+import com.google.inject.Singleton;
+
+
+/** REST end point for Timeline Reader */
+@Private
+@Unstable
+@Singleton
+@Path("/ws/v2/timeline")
+public class TimelineReaderWebServices {
+
+ private void init(HttpServletResponse response) {
+ response.setContentType(null);
+ }
+
+ /**
+ * Return the description of the timeline reader web services.
+ */
+ @GET
+ @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ public TimelineAbout about(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res) {
+ init(res);
+ return TimelineUtils.createTimelineAbout("Timeline Reader API");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
new file mode 100644
index 0000000..7098814
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertEquals;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.Service.STATE;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer;
+import org.junit.Test;
+
+public class TestTimelineReaderServer {
+
+ @Test(timeout = 60000)
+ public void testStartStopServer() throws Exception {
+ TimelineReaderServer server = new TimelineReaderServer();
+ Configuration config = new YarnConfiguration();
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ try {
+ server.init(config);
+ assertEquals(STATE.INITED, server.getServiceState());
+ assertEquals(2, server.getServices().size());
+
+ server.start();
+ assertEquals(STATE.STARTED, server.getServiceState());
+
+ server.stop();
+ assertEquals(STATE.STOPPED, server.getServiceState());
+ } finally {
+ server.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e281987f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
new file mode 100644
index 0000000..a9145d0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.IOException;
+import java.lang.reflect.UndeclaredThrowableException;
+import java.net.HttpURLConnection;
+import java.net.URI;
+import java.net.URL;
+
+import javax.ws.rs.core.MediaType;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.config.ClientConfig;
+import com.sun.jersey.api.client.config.DefaultClientConfig;
+import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
+import com.sun.jersey.client.urlconnection.URLConnectionClientHandler;
+
+public class TestTimelineReaderWebServices {
+ private int serverPort;
+ private TimelineReaderServer server;
+
+ @Before
+ public void init() throws Exception {
+ try {
+ Configuration config = new YarnConfiguration();
+ config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ "localhost:0");
+ server = new TimelineReaderServer();
+ server.init(config);
+ server.start();
+ serverPort = server.getWebServerPort();
+ } catch (Exception e) {
+ Assert.fail("Web server failed to start");
+ }
+ }
+
+ @After
+ public void stop() throws Exception {
+ if (server != null) {
+ server.stop();
+ server = null;
+ }
+ }
+
+ private static Client createClient() {
+ ClientConfig cfg = new DefaultClientConfig();
+ cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
+ return new Client(new URLConnectionClientHandler(
+ new DummyURLConnectionFactory()), cfg);
+ }
+
+ private static ClientResponse getResponse(Client client, URI uri) throws Exception {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ if (resp == null ||
+ resp.getClientResponseStatus() != ClientResponse.Status.OK) {
+ System.out.println(resp.getClientResponseStatus());
+ throw new IOException("Incorrect response from timeline reader.");
+ }
+ return resp;
+ }
+
+ private static class DummyURLConnectionFactory
+ implements HttpURLConnectionFactory {
+
+ @Override
+ public HttpURLConnection getHttpURLConnection(final URL url) throws IOException {
+ try {
+ return (HttpURLConnection)url.openConnection();
+ } catch (UndeclaredThrowableException e) {
+ throw new IOException(e.getCause());
+ }
+ }
+ }
+
+ @Test
+ public void testAbout()
+ throws IOException {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/");
+ Client client = createClient();
+ try {
+ ClientResponse resp = getResponse(client, uri);
+ TimelineAbout about = resp.getEntity(TimelineAbout.class);
+ Assert.assertNotNull(about);
+ Assert.assertEquals("Timeline Reader API", about.getAbout());
+ } catch (Exception re) {
+ throw new IOException(
+ "Failed to get the response from timeline reader.", re);
+ } finally {
+ client.destroy();
+ }
+ }
+}
\ No newline at end of file
[19/50] [abbrv] hadoop git commit: YARN-3529. Added mini HBase
cluster and Phoenix support to timeline service v2 unit tests. Contributed by
Li Lu.
Posted by sj...@apache.org.
YARN-3529. Added mini HBase cluster and Phoenix support to timeline service v2 unit tests. Contributed by Li Lu.
(cherry picked from commit 9b0ae9363dbce36f70aa92415a1178f65f3c1dcd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a68cdab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a68cdab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a68cdab
Branch: refs/heads/YARN-2928
Commit: 7a68cdab24df2d584270227beaea8971d94d66c1
Parents: 9b794ab
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue May 12 13:53:38 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:11 2015 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 52 ++++++++++++++++
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn-server-timelineservice/pom.xml | 47 ++++++++++----
.../storage/PhoenixTimelineWriterImpl.java | 22 ++++++-
.../storage/TestPhoenixTimelineWriterImpl.java | 65 ++++++++++++++------
5 files changed, 155 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a68cdab/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index acc021d..7a35939 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -45,6 +45,9 @@
<!-- Version number for xerces used by JDiff -->
<xerces.jdiff.version>2.11.0</xerces.jdiff.version>
+ <hbase.version>1.0.1</hbase.version>
+ <phoenix.version>4.5.0-SNAPSHOT</phoenix.version>
+
<hadoop.assemblies.version>${project.version}</hadoop.assemblies.version>
<commons-daemon.version>1.0.13</commons-daemon.version>
@@ -993,6 +996,55 @@
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <version>${phoenix.version}</version>
+ <exclusions>
+ <!-- Exclude jline from here -->
+ <exclusion>
+ <artifactId>jline</artifactId>
+ <groupId>jline</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <type>test-jar</type>
+ <version>${phoenix.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.jruby</groupId>
+ <artifactId>jruby-complete</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a68cdab/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a290c22..0b06502 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -67,6 +67,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3134. Implemented Phoenix timeline writer to access HBase backend. (Li
Lu via zjshen)
+ YARN-3529. Added mini HBase cluster and Phoenix support to timeline service
+ v2 unit tests. (Li Lu via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a68cdab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index f62230f..1e914de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -122,21 +122,44 @@
</dependency>
<dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <version>4.3.0</version>
- <exclusions>
- <!-- Exclude jline from here -->
- <exclusion>
- <artifactId>jline</artifactId>
- <groupId>jline</groupId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
</dependency>
<dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ </dependency>
+ <!-- for unit tests only -->
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <scope>test</scope>
+ <optional>true</optional>
+ </dependency>
+
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a68cdab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
index af8a233..5b4442c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
@@ -43,6 +44,7 @@ import java.sql.Statement;
import java.util.Collection;
import java.util.Iterator;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
@Private
@@ -50,6 +52,13 @@ import java.util.Set;
public class PhoenixTimelineWriterImpl extends AbstractService
implements TimelineWriter {
+ public static final String TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR
+ = YarnConfiguration.TIMELINE_SERVICE_PREFIX
+ + "writer.phoenix.connectionString";
+
+ public static final String TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT
+ = "jdbc:phoenix:localhost:2181:/hbase";
+
private static final Log LOG
= LogFactory.getLog(PhoenixTimelineWriterImpl.class);
private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
@@ -90,7 +99,10 @@ public class PhoenixTimelineWriterImpl extends AbstractService
private static final String PHOENIX_STORAGE_SEPARATOR = ";";
/** Connection string to the deployed Phoenix cluster */
- static final String CONN_STRING = "jdbc:phoenix:localhost:2181:/hbase";
+ @VisibleForTesting
+ String connString = null;
+ @VisibleForTesting
+ Properties connProperties = new Properties();
PhoenixTimelineWriterImpl() {
super((PhoenixTimelineWriterImpl.class.getName()));
@@ -98,6 +110,10 @@ public class PhoenixTimelineWriterImpl extends AbstractService
@Override
protected void serviceInit(Configuration conf) throws Exception {
+ // so check it here and only read in the config if it's not overridden.
+ connString =
+ conf.get(TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
+ TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT);
createTables();
super.init(conf);
}
@@ -174,11 +190,11 @@ public class PhoenixTimelineWriterImpl extends AbstractService
// Utility functions
@Private
@VisibleForTesting
- static Connection getConnection() throws IOException {
+ Connection getConnection() throws IOException {
Connection conn;
try {
Class.forName(DRIVER_CLASS_NAME);
- conn = DriverManager.getConnection(CONN_STRING);
+ conn = DriverManager.getConnection(connString, connProperties);
conn.setAutoCommit(false);
} catch (SQLException se) {
LOG.error("Failed to connect to phoenix server! "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a68cdab/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
index a55893e..dece83d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
@@ -23,30 +23,37 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
-public class TestPhoenixTimelineWriterImpl {
- private PhoenixTimelineWriterImpl writer;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
- @Before
- public void setup() throws Exception {
- // TODO: launch a miniphoenix cluster, or else we're directly operating on
- // the active Phoenix cluster
+public class TestPhoenixTimelineWriterImpl extends BaseTest {
+ private static PhoenixTimelineWriterImpl writer;
+ private static final int BATCH_SIZE = 3;
+
+ @BeforeClass
+ public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
- writer = createPhoenixWriter(conf);
+ writer = setupPhoenixClusterAndWriterForTest(conf);
}
- @Ignore
- @Test
+ @Test(timeout = 90000)
public void testPhoenixWriterBasic() throws Exception {
// Set up a list of timeline entities and write them back to Phoenix
int numEntity = 12;
@@ -91,28 +98,48 @@ public class TestPhoenixTimelineWriterImpl {
verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
}
- @After
- public void cleanup() throws Exception {
- // Note: it is assumed that we're working on a test only cluster, or else
- // this cleanup process will drop the entity table.
+ @AfterClass
+ public static void cleanup() throws Exception {
writer.dropTable(PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME);
writer.dropTable(PhoenixTimelineWriterImpl.EVENT_TABLE_NAME);
writer.dropTable(PhoenixTimelineWriterImpl.METRIC_TABLE_NAME);
writer.serviceStop();
+ tearDownMiniCluster();
}
- private static PhoenixTimelineWriterImpl createPhoenixWriter(
+ private static PhoenixTimelineWriterImpl setupPhoenixClusterAndWriterForTest(
YarnConfiguration conf) throws Exception{
+ Map<String, String> props = new HashMap<>();
+ // Must update config before starting server
+ props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
+ Boolean.FALSE.toString());
+ props.put("java.security.krb5.realm", "");
+ props.put("java.security.krb5.kdc", "");
+ props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER,
+ Boolean.FALSE.toString());
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
+ props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
+ // Make a small batch size to test multiple calls to reserve sequences
+ props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB,
+ Long.toString(BATCH_SIZE));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+
PhoenixTimelineWriterImpl myWriter = new PhoenixTimelineWriterImpl();
+ // Change connection settings for test
+ conf.set(
+ PhoenixTimelineWriterImpl.TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
+ getUrl());
+ myWriter.connProperties = PropertiesUtil.deepCopy(TEST_PROPERTIES);
myWriter.serviceInit(conf);
return myWriter;
}
private void verifySQLWithCount(String sql, int targetCount, String message)
- throws Exception{
+ throws Exception {
try (
Statement stmt =
- PhoenixTimelineWriterImpl.getConnection().createStatement();
+ writer.getConnection().createStatement();
ResultSet rs = stmt.executeQuery(sql)) {
assertTrue("Result set empty on statement " + sql, rs.next());
assertNotNull("Fail to execute query " + sql, rs);
[41/50] [abbrv] hadoop git commit: YARN-3049. [Storage
Implementation] Implement storage reader interface to fetch raw data from
HBase backend (Zhijie Shen via sjlee)
Posted by sj...@apache.org.
YARN-3049. [Storage Implementation] Implement storage reader interface to fetch raw data from HBase backend (Zhijie Shen via sjlee)
(cherry picked from commit 07433c2ad52df9e844dbd90020c277d3df844dcd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bdf34f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bdf34f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bdf34f5
Branch: refs/heads/YARN-2928
Commit: 4bdf34f5b1f17e49e5dcf67b219aec65f4608457
Parents: d1a1a22
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Aug 7 10:00:22 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 6 +
.../records/timelineservice/TimelineEntity.java | 9 +-
.../storage/FileSystemTimelineReaderImpl.java | 164 +++----
.../storage/HBaseTimelineReaderImpl.java | 424 +++++++++++++++++++
.../storage/HBaseTimelineWriterImpl.java | 43 +-
.../storage/TimelineSchemaCreator.java | 12 +
.../storage/apptoflow/AppToFlowColumn.java | 126 ++++++
.../apptoflow/AppToFlowColumnFamily.java | 51 +++
.../storage/apptoflow/AppToFlowRowKey.java | 39 ++
.../storage/apptoflow/AppToFlowTable.java | 110 +++++
.../storage/apptoflow/package-info.java | 23 +
.../storage/common/BaseTable.java | 16 +
.../storage/common/ColumnPrefix.java | 2 +-
.../common/TimelineEntitySchemaConstants.java | 68 ---
.../common/TimelineHBaseSchemaConstants.java | 68 +++
.../storage/common/TimelineReaderUtils.java | 112 +++++
.../storage/entity/EntityColumn.java | 2 +-
.../storage/entity/EntityColumnFamily.java | 2 +-
.../storage/entity/EntityColumnPrefix.java | 2 +-
.../storage/entity/EntityRowKey.java | 36 +-
.../storage/entity/EntityTable.java | 8 +-
.../storage/TestHBaseTimelineWriterImpl.java | 82 +++-
23 files changed, 1198 insertions(+), 210 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a484040..2e06793 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -82,6 +82,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3949. Ensure timely flush of timeline writes. (Sangjin Lee via
junping_du)
+ YARN-3049. [Storage Implementation] Implement storage reader interface to
+ fetch raw data from HBase backend (Zhijie Shen via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index d25d1d9..5583cd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -506,4 +506,10 @@
</Or>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
+
+ <!-- Object cast is based on the event type -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 9ef2d90..0701001 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -29,7 +29,9 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.NavigableSet;
import java.util.Set;
+import java.util.TreeSet;
/**
* The basic timeline entity data structure for timeline service v2. Timeline
@@ -133,7 +135,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
private HashMap<String, Object> info = new HashMap<>();
private HashMap<String, String> configs = new HashMap<>();
private Set<TimelineMetric> metrics = new HashSet<>();
- private Set<TimelineEvent> events = new HashSet<>();
+ // events should be sorted by timestamp in descending order
+ private NavigableSet<TimelineEvent> events = new TreeSet<>();
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
private HashMap<String, Set<String>> relatesToEntities = new HashMap<>();
private long createdTime;
@@ -334,7 +337,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
@XmlElement(name = "events")
- public Set<TimelineEvent> getEvents() {
+ public NavigableSet<TimelineEvent> getEvents() {
if (real == null) {
return events;
} else {
@@ -342,7 +345,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setEvents(Set<TimelineEvent> events) {
+ public void setEvents(NavigableSet<TimelineEvent> events) {
if (real == null) {
this.events = events;
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index f9f1d1d..45ddd1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.JsonGenerationException;
import org.codehaus.jackson.map.JsonMappingException;
@@ -119,59 +120,32 @@ public class FileSystemTimelineReaderImpl extends AbstractService
private static void fillFields(TimelineEntity finalEntity,
TimelineEntity real, EnumSet<Field> fields) {
if (fields.contains(Field.ALL)) {
- finalEntity.setConfigs(real.getConfigs());
- finalEntity.setMetrics(real.getMetrics());
- finalEntity.setInfo(real.getInfo());
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- finalEntity.setEvents(real.getEvents());
- return;
+ fields = EnumSet.allOf(Field.class);
}
for (Field field : fields) {
switch(field) {
- case CONFIGS:
- finalEntity.setConfigs(real.getConfigs());
- break;
- case METRICS:
- finalEntity.setMetrics(real.getMetrics());
- break;
- case INFO:
- finalEntity.setInfo(real.getInfo());
- break;
- case IS_RELATED_TO:
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- break;
- case RELATES_TO:
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- break;
- case EVENTS:
- finalEntity.setEvents(real.getEvents());
- break;
- default:
- continue;
- }
- }
- }
-
- private static boolean matchFilter(Object infoValue, Object filterValue) {
- return infoValue.equals(filterValue);
- }
-
- private static boolean matchFilters(Map<String, ? extends Object> entityInfo,
- Map<String, ? extends Object> filters) {
- if (entityInfo == null || entityInfo.isEmpty()) {
- return false;
- }
- for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
- Object infoValue = entityInfo.get(filter.getKey());
- if (infoValue == null) {
- return false;
- }
- if (!matchFilter(infoValue, filter.getValue())) {
- return false;
+ case CONFIGS:
+ finalEntity.setConfigs(real.getConfigs());
+ break;
+ case METRICS:
+ finalEntity.setMetrics(real.getMetrics());
+ break;
+ case INFO:
+ finalEntity.setInfo(real.getInfo());
+ break;
+ case IS_RELATED_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case RELATES_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case EVENTS:
+ finalEntity.setEvents(real.getEvents());
+ break;
+ default:
+ continue;
}
}
- return true;
}
private String getFlowRunPath(String userId, String clusterId, String flowId,
@@ -186,10 +160,10 @@ public class FileSystemTimelineReaderImpl extends AbstractService
String appFlowMappingFile = rootPath + "/" + ENTITIES_DIR + "/" +
clusterId + "/" + APP_FLOW_MAPPING_FILE;
try (BufferedReader reader =
- new BufferedReader(new InputStreamReader(
- new FileInputStream(
- appFlowMappingFile), Charset.forName("UTF-8")));
- CSVParser parser = new CSVParser(reader, csvFormat)) {
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(
+ appFlowMappingFile), Charset.forName("UTF-8")));
+ CSVParser parser = new CSVParser(reader, csvFormat)) {
for (CSVRecord record : parser.getRecords()) {
if (record.size() < 4) {
continue;
@@ -207,36 +181,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
throw new IOException("Unable to get flow info");
}
- private static boolean matchMetricFilters(Set<TimelineMetric> metrics,
- Set<String> metricFilters) {
- Set<String> tempMetrics = new HashSet<String>();
- for (TimelineMetric metric : metrics) {
- tempMetrics.add(metric.getId());
- }
-
- for (String metricFilter : metricFilters) {
- if (!tempMetrics.contains(metricFilter)) {
- return false;
- }
- }
- return true;
- }
-
- private static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
- Set<String> eventFilters) {
- Set<String> tempEvents = new HashSet<String>();
- for (TimelineEvent event : entityEvents) {
- tempEvents.add(event.getId());
- }
-
- for (String eventFilter : eventFilters) {
- if (!tempEvents.contains(eventFilter)) {
- return false;
- }
- }
- return true;
- }
-
private static TimelineEntity createEntityToBeReturned(TimelineEntity entity,
EnumSet<Field> fieldsToRetrieve) {
TimelineEntity entityToBeReturned = new TimelineEntity();
@@ -254,23 +198,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
return (time >= timeBegin) && (time <= timeEnd);
}
- private static boolean matchRelations(
- Map<String, Set<String>> entityRelations,
- Map<String, Set<String>> relations) {
- for (Map.Entry<String, Set<String>> relation : relations.entrySet()) {
- Set<String> ids = entityRelations.get(relation.getKey());
- if (ids == null) {
- return false;
- }
- for (String id : relation.getValue()) {
- if (!ids.contains(id)) {
- return false;
- }
- }
- }
- return true;
- }
-
private static void mergeEntities(TimelineEntity entity1,
TimelineEntity entity2) {
// Ideally created time wont change except in the case of issue from client.
@@ -364,22 +291,22 @@ public class FileSystemTimelineReaderImpl extends AbstractService
// First sort the selected entities based on created/start time.
Map<Long, Set<TimelineEntity>> sortedEntities =
new TreeMap<>(
- new Comparator<Long>() {
- @Override
- public int compare(Long l1, Long l2) {
- return l2.compareTo(l1);
+ new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return l2.compareTo(l1);
+ }
}
- }
);
for (File entityFile : dir.listFiles()) {
if (!entityFile.getName().contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
continue;
}
try (BufferedReader reader =
- new BufferedReader(
- new InputStreamReader(
- new FileInputStream(
- entityFile), Charset.forName("UTF-8")))) {
+ new BufferedReader(
+ new InputStreamReader(
+ new FileInputStream(
+ entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
if (!entity.getType().equals(entityType)) {
continue;
@@ -393,27 +320,32 @@ public class FileSystemTimelineReaderImpl extends AbstractService
continue;
}
if (relatesTo != null && !relatesTo.isEmpty() &&
- !matchRelations(entity.getRelatesToEntities(), relatesTo)) {
+ !TimelineReaderUtils
+ .matchRelations(entity.getRelatesToEntities(), relatesTo)) {
continue;
}
if (isRelatedTo != null && !isRelatedTo.isEmpty() &&
- !matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
+ !TimelineReaderUtils
+ .matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
continue;
}
if (infoFilters != null && !infoFilters.isEmpty() &&
- !matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
continue;
}
if (configFilters != null && !configFilters.isEmpty() &&
- !matchFilters(entity.getConfigs(), configFilters)) {
+ !TimelineReaderUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
continue;
}
if (metricFilters != null && !metricFilters.isEmpty() &&
- !matchMetricFilters(entity.getMetrics(), metricFilters)) {
+ !TimelineReaderUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
continue;
}
if (eventFilters != null && !eventFilters.isEmpty() &&
- !matchEventFilters(entity.getEvents(), eventFilters)) {
+ !TimelineReaderUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
continue;
}
TimelineEntity entityToBeReturned =
@@ -461,8 +393,8 @@ public class FileSystemTimelineReaderImpl extends AbstractService
File entityFile =
new File(dir, entityId + TIMELINE_SERVICE_STORAGE_EXTENSION);
try (BufferedReader reader =
- new BufferedReader(new InputStreamReader(
- new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
return createEntityToBeReturned(entity, fieldsToRetrieve);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
new file mode 100644
index 0000000..5258b9c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -0,0 +1,424 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+
+import com.google.common.base.Preconditions;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+public class HBaseTimelineReaderImpl
+ extends AbstractService implements TimelineReader {
+
+ private static final Log LOG = LogFactory
+ .getLog(HBaseTimelineReaderImpl.class);
+ private static final long DEFAULT_BEGIN_TIME = 0L;
+ private static final long DEFAULT_END_TIME = Long.MAX_VALUE;
+
+ private Configuration hbaseConf = null;
+ private Connection conn;
+ private EntityTable entityTable;
+ private AppToFlowTable appToFlowTable;
+
+ public HBaseTimelineReaderImpl() {
+ super(HBaseTimelineReaderImpl.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ hbaseConf = HBaseConfiguration.create(conf);
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ entityTable = new EntityTable();
+ appToFlowTable = new AppToFlowTable();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public TimelineEntity getEntity(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve)
+ throws IOException {
+ validateParams(userId, clusterId, appId, entityType, entityId, true);
+ // In reality both should be null or neither should be null
+ if (flowId == null || flowRunId == null) {
+ FlowContext context = lookupFlowContext(clusterId, appId);
+ flowId = context.flowId;
+ flowRunId = context.flowRunId;
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+
+ byte[] rowKey = EntityRowKey.getRowKey(
+ clusterId, userId, flowId, flowRunId, appId, entityType, entityId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ return parseEntity(
+ entityTable.getResult(hbaseConf, conn, get), fieldsToRetrieve,
+ false, DEFAULT_BEGIN_TIME, DEFAULT_END_TIME, false, DEFAULT_BEGIN_TIME,
+ DEFAULT_END_TIME, null, null, null, null, null, null);
+ }
+
+ @Override
+ public Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ validateParams(userId, clusterId, appId, entityType, null, false);
+ // In reality both should be null or neither should be null
+ if (flowId == null || flowRunId == null) {
+ FlowContext context = lookupFlowContext(clusterId, appId);
+ flowId = context.flowId;
+ flowRunId = context.flowRunId;
+ }
+ if (limit == null) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (modifiedTimeBegin == null) {
+ modifiedTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (modifiedTimeEnd == null) {
+ modifiedTimeEnd = DEFAULT_END_TIME;
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+
+ NavigableSet<TimelineEntity> entities = new TreeSet<>();
+ // Scan through part of the table to find the entities belong to one app and
+ // one type
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
+ clusterId, userId, flowId, flowRunId, appId, entityType));
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner = entityTable.getResultScanner(hbaseConf, conn, scan);
+ for (Result result : scanner) {
+ TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
+ true, createdTimeBegin, createdTimeEnd,
+ true, modifiedTimeBegin, modifiedTimeEnd,
+ isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
+ metricFilters);
+ if (entity == null) {
+ continue;
+ }
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ entities.add(entity);
+ }
+ return entities;
+ }
+
+ private FlowContext lookupFlowContext(String clusterId, String appId)
+ throws IOException {
+ byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
+ Get get = new Get(rowKey);
+ Result result = appToFlowTable.getResult(hbaseConf, conn, get);
+ if (result != null && !result.isEmpty()) {
+ return new FlowContext(
+ AppToFlowColumn.FLOW_ID.readResult(result).toString(),
+ ((Number) AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
+ } else {
+ throw new IOException(
+ "Unable to find the context flow ID and flow run ID for clusterId=" +
+ clusterId + ", appId=" + appId);
+ }
+ }
+
+ private static class FlowContext {
+ private String flowId;
+ private Long flowRunId;
+ public FlowContext(String flowId, Long flowRunId) {
+ this.flowId = flowId;
+ this.flowRunId = flowRunId;
+ }
+ }
+
+ private static void validateParams(String userId, String clusterId,
+ String appId, String entityType, String entityId, boolean checkEntityId) {
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ if (checkEntityId) {
+ Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
+ }
+ }
+
+ private static TimelineEntity parseEntity(
+ Result result, EnumSet<Field> fieldsToRetrieve,
+ boolean checkCreatedTime, long createdTimeBegin, long createdTimeEnd,
+ boolean checkModifiedTime, long modifiedTimeBegin, long modifiedTimeEnd,
+ Map<String, Set<String>> isRelatedTo, Map<String, Set<String>> relatesTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> eventFilters, Set<String> metricFilters)
+ throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ entity.setType(EntityColumn.TYPE.readResult(result).toString());
+ entity.setId(EntityColumn.ID.readResult(result).toString());
+
+ // fetch created time
+ entity.setCreatedTime(
+ ((Number) EntityColumn.CREATED_TIME.readResult(result)).longValue());
+ if (checkCreatedTime && (entity.getCreatedTime() < createdTimeBegin ||
+ entity.getCreatedTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // fetch modified time
+ entity.setCreatedTime(
+ ((Number) EntityColumn.MODIFIED_TIME.readResult(result)).longValue());
+ if (checkModifiedTime && (entity.getModifiedTime() < modifiedTimeBegin ||
+ entity.getModifiedTime() > modifiedTimeEnd)) {
+ return null;
+ }
+
+ // fetch is related to entities
+ boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO);
+ if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
+ entity.getIsRelatedToEntities(), isRelatedTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities
+ boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO);
+ if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
+ entity.getRelatesToEntities(), relatesTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info
+ boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO);
+ if (checkInfo &&
+ !TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.INFO)) {
+ entity.getInfo().clear();
+ }
+ }
+
+ // fetch configs
+ boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG);
+ if (checkConfigs && !TimelineReaderUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.CONFIGS)) {
+ entity.getConfigs().clear();
+ }
+ }
+
+ // fetch events
+ boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
+ readEvents(entity, result);
+ if (checkEvents && !TimelineReaderUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics
+ boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ readMetrics(entity, result);
+ if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.METRICS)) {
+ entity.getMetrics().clear();
+ }
+ }
+ return entity;
+ }
+
+ private static void readRelationship(
+ TimelineEntity entity, Result result, EntityColumnPrefix prefix)
+ throws IOException {
+ // isRelatedTo and relatesTo are of type Map<String, Set<String>>
+ Map<String, Object> columns = prefix.readResults(result);
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ for (String id : Separator.VALUES.splitEncoded(
+ column.getValue().toString())) {
+ if (prefix.equals(EntityColumnPrefix.IS_RELATED_TO)) {
+ entity.addIsRelatedToEntity(column.getKey(), id);
+ } else {
+ entity.addRelatesToEntity(column.getKey(), id);
+ }
+ }
+ }
+ }
+
+ private static void readKeyValuePairs(
+ TimelineEntity entity, Result result, EntityColumnPrefix prefix)
+ throws IOException {
+ // info and configuration are of type Map<String, Object or String>
+ Map<String, Object> columns = prefix.readResults(result);
+ if (prefix.equals(EntityColumnPrefix.CONFIG)) {
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ entity.addConfig(column.getKey(), column.getKey().toString());
+ }
+ } else {
+ entity.addInfo(columns);
+ }
+ }
+
+ private static void readEvents(TimelineEntity entity, Result result)
+ throws IOException {
+ Map<String, TimelineEvent> eventsMap = new HashMap<>();
+ Map<String, Object> eventsResult =
+ EntityColumnPrefix.EVENT.readResults(result);
+ for (Map.Entry<String,Object> eventResult : eventsResult.entrySet()) {
+ Collection<String> tokens =
+ Separator.VALUES.splitEncoded(eventResult.getKey());
+ if (tokens.size() != 2 && tokens.size() != 3) {
+ throw new IOException(
+ "Invalid event column name: " + eventResult.getKey());
+ }
+ Iterator<String> idItr = tokens.iterator();
+ String id = idItr.next();
+ String tsStr = idItr.next();
+ // TODO: timestamp is not correct via ser/des through UTF-8 string
+ Long ts =
+ TimelineWriterUtils.invert(Bytes.toLong(tsStr.getBytes(
+ StandardCharsets.UTF_8)));
+ String key = Separator.VALUES.joinEncoded(id, ts.toString());
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ if (tokens.size() == 3) {
+ String infoKey = idItr.next();
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ }
+ Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
+ entity.addEvents(eventsSet);
+ }
+
+ private static void readMetrics(TimelineEntity entity, Result result)
+ throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
+ metricsResult.entrySet()) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(metricResult.getKey());
+ // Simply assume that if the value set contains more than 1 elements, the
+ // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
+ metric.setType(metricResult.getValue().size() > 1 ?
+ TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE);
+ metric.addValues(metricResult.getValue());
+ entity.addMetric(metric);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 3173e87..5290415 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -33,9 +33,14 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
@@ -55,6 +60,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private Connection conn;
private TypedBufferedMutator<EntityTable> entityTable;
+ private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
private static final Log LOG = LogFactory
.getLog(HBaseTimelineWriterImpl.class);
@@ -77,6 +83,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
Configuration hbaseConf = HBaseConfiguration.create(conf);
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
+ appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -97,7 +104,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
byte[] rowKey =
EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
- te);
+ te.getType(), te.getId());
storeInfo(rowKey, te, flowVersion);
storeEvents(rowKey, te.getEvents());
@@ -107,11 +114,37 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
EntityColumnPrefix.IS_RELATED_TO);
storeRelations(rowKey, te.getRelatesToEntities(),
EntityColumnPrefix.RELATES_TO);
- }
+ if (isApplicationCreated(te)) {
+ onApplicationCreated(
+ clusterId, userId, flowName, flowVersion, flowRunId, appId, te);
+ }
+ }
return putStatus;
}
+ private static boolean isApplicationCreated(TimelineEntity te) {
+ if (te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) {
+ boolean isAppCreated = false;
+ for (TimelineEvent event : te.getEvents()) {
+ if (event.getId().equals(
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private void onApplicationCreated(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntity te) throws IOException {
+ byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
+ AppToFlowColumn.FLOW_ID.store(rowKey, appToFlowTable, null, flowName);
+ AppToFlowColumn.FLOW_RUN_ID.store(
+ rowKey, appToFlowTable, null, flowRunId);
+ }
+
/**
* Stores the Relations from the {@linkplain TimelineEntity} object
*/
@@ -245,6 +278,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
public void flush() throws IOException {
// flush all buffered mutators
entityTable.flush();
+ appToFlowTable.flush();
}
/**
@@ -258,6 +292,11 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// The close API performs flushing and releases any resources held
entityTable.close();
}
+ if (appToFlowTable != null) {
+ LOG.info("closing app_flow table");
+ // The close API performs flushing and releases any resources held
+ appToFlowTable.close();
+ }
if (conn != null) {
LOG.info("closing the hbase Connection");
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index a5cc2ab..2c3897d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
/**
@@ -70,6 +71,11 @@ public class TimelineSchemaCreator {
int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
}
+ // Grab the appToflowTableName argument
+ String appToflowTableName = commandLine.getOptionValue("a2f");
+ if (StringUtils.isNotBlank(appToflowTableName)) {
+ hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
+ }
createAllTables(hbaseConf);
}
@@ -95,6 +101,11 @@ public class TimelineSchemaCreator {
o.setRequired(false);
options.addOption(o);
+ o = new Option("a2f", "appToflowTableName", true, "app to flow table name");
+ o.setArgName("appToflowTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
CommandLineParser parser = new PosixParser();
CommandLine commandLine = null;
try {
@@ -120,6 +131,7 @@ public class TimelineSchemaCreator {
throw new IOException("Cannot create table since admin is null");
}
new EntityTable().createTable(admin, hbaseConf);
+ new AppToFlowTable().createTable(admin, hbaseConf);
} finally {
if (conn != null) {
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
new file mode 100644
index 0000000..423037a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+import java.io.IOException;
+
+/**
+ * Identifies fully qualified columns for the {@link AppToFlowTable}.
+ */
+public enum AppToFlowColumn implements Column<AppToFlowTable> {
+
+ /**
+ * The flow ID
+ */
+ FLOW_ID(AppToFlowColumnFamily.MAPPING, "flow_id"),
+
+ /**
+ * The flow run ID
+ */
+ FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id");
+
+ private final ColumnHelper<AppToFlowTable> column;
+ private final ColumnFamily<AppToFlowTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+
+ AppToFlowColumn(ColumnFamily<AppToFlowTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.column = new ColumnHelper<AppToFlowTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<AppToFlowTable> tableMutator, Long timestamp,
+ Object inputValue) throws IOException {
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link AppToFlowColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier Name of the column to retrieve
+ * @return the corresponding {@link AppToFlowColumn} or null
+ */
+ public static final AppToFlowColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (AppToFlowColumn ec : AppToFlowColumn.values()) {
+ // Find a match based only on name.
+ if (ec.getColumnQualifier().equals(columnQualifier)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link AppToFlowColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param name Name of the column to retrieve
+ * @return the corresponding {@link AppToFlowColumn} or null if both arguments
+ * don't match.
+ */
+ public static final AppToFlowColumn columnFor(
+ AppToFlowColumnFamily columnFamily, String name) {
+
+ for (AppToFlowColumn ec : AppToFlowColumn.values()) {
+ // Find a match based column family and on name.
+ if (ec.columnFamily.equals(columnFamily)
+ && ec.getColumnQualifier().equals(name)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
new file mode 100644
index 0000000..e74235f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the app_flow table column families.
+ */
+public enum AppToFlowColumnFamily implements ColumnFamily<AppToFlowTable> {
+ /**
+ * Mapping column family houses known columns such as flowId and flowRunId
+ */
+ MAPPING("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ AppToFlowColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
new file mode 100644
index 0000000..ad4fec6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the app_flow table.
+ */
+public class AppToFlowRowKey {
+ /**
+ * Constructs a row key prefix for the app_flow table as follows:
+ * {@code clusterId!AppId}
+ *
+ * @param clusterId
+ * @param appId
+ * @return byte array with the row key
+ */
+ public static byte[] getRowKey(String clusterId, String appId) {
+ return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, appId));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
new file mode 100644
index 0000000..2467856
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+
+import java.io.IOException;
+
+/**
+ * The app_flow table as column families mapping. Mapping stores
+ * appId to flowId and flowRunId mapping information
+ *
+ * Example app_flow table record:
+ *
+ * <pre>
+ * |--------------------------------------|
+ * | Row | Column Family |
+ * | key | info |
+ * |--------------------------------------|
+ * | clusterId! | flowId: |
+ * | AppId | foo@daily_hive_report |
+ * | | |
+ * | | flowRunId: |
+ * | | 1452828720457 |
+ * | | |
+ * | | |
+ * | | |
+ * |--------------------------------------|
+ * </pre>
+ */
+public class AppToFlowTable extends BaseTable<AppToFlowTable> {
+ /** app_flow prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "app-flow";
+
+ /** config param name that specifies the app_flow table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /** default value for app_flow table name */
+ private static final String DEFAULT_TABLE_NAME = "timelineservice.app_flow";
+
+ private static final Log LOG = LogFactory.getLog(AppToFlowTable.class);
+
+ public AppToFlowTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor appToFlowTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor mappCF =
+ new HColumnDescriptor(AppToFlowColumnFamily.MAPPING.getBytes());
+ mappCF.setBloomFilterType(BloomType.ROWCOL);
+ appToFlowTableDescp.addFamily(mappCF);
+
+ appToFlowTableDescp
+ .setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ appToFlowTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(appToFlowTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
new file mode 100644
index 0000000..df7ffc1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
index e8d8b5c..abba79a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -24,6 +24,8 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
@@ -94,6 +96,20 @@ public abstract class BaseTable<T> {
}
/**
+ *
+ * @param hbaseConf used to read settings that override defaults
+ * @param conn used to create table from
+ * @param get that specifies what single row you want to get from this table
+ * @return result of get operation
+ * @throws IOException
+ */
+ public Result getResult(Configuration hbaseConf, Connection conn, Get get)
+ throws IOException {
+ Table table = conn.getTable(getTableName(hbaseConf));
+ return table.get(get);
+ }
+
+ /**
* Get the table name for this table.
*
* @param hbaseConf
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index 671c824..509ff49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -64,7 +64,7 @@ public interface ColumnPrefix<T> {
public Object readResult(Result result, String qualifier) throws IOException;
/**
- * @param resultfrom which to read columns
+ * @param result from which to read columns
* @return the latest values of columns in the column family with this prefix
* (or all of them if the prefix value is null).
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
deleted file mode 100644
index 5518a27..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * contains the constants used in the context of schema accesses for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * information
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TimelineEntitySchemaConstants {
-
- /**
- * Used to create a pre-split for tables starting with a username in the
- * prefix. TODO: this may have to become a config variable (string with
- * separators) so that different installations can presplit based on their own
- * commonly occurring names.
- */
- private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
- Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
- Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
- Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
- Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
- Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
- Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
- Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
- Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
- Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
- Bytes.toBytes("z") };
-
- /**
- * The length at which keys auto-split
- */
- public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
-
- /**
- * @return splits for splits where a user is a prefix.
- */
- public final static byte[][] getUsernameSplits() {
- byte[][] kloon = USERNAME_SPLITS.clone();
- // Deep copy.
- for (int row = 0; row < USERNAME_SPLITS.length; row++) {
- kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
- }
- return kloon;
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
new file mode 100644
index 0000000..bbf498a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineHBaseSchemaConstants {
+
+ /**
+ * Used to create a pre-split for tables starting with a username in the
+ * prefix. TODO: this may have to become a config variable (string with
+ * separators) so that different installations can presplit based on their own
+ * commonly occurring names.
+ */
+ private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
+ Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
+ Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
+ Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
+ Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
+ Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
+ Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
+ Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
+ Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
+ Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
+ Bytes.toBytes("z") };
+
+ /**
+ * The length at which keys auto-split
+ */
+ public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ /**
+ * @return splits for splits where a user is a prefix.
+ */
+ public final static byte[][] getUsernameSplits() {
+ byte[][] kloon = USERNAME_SPLITS.clone();
+ // Deep copy.
+ for (int row = 0; row < USERNAME_SPLITS.length; row++) {
+ kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
+ }
+ return kloon;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
new file mode 100644
index 0000000..91d7ba4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineReaderUtils.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+
+public class TimelineReaderUtils {
+ /**
+ *
+ * @param entityRelations the relations of an entity
+ * @param relationFilters the relations for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchRelations(
+ Map<String, Set<String>> entityRelations,
+ Map<String, Set<String>> relationFilters) {
+ for (Map.Entry<String, Set<String>> relation : relationFilters.entrySet()) {
+ Set<String> ids = entityRelations.get(relation.getKey());
+ if (ids == null) {
+ return false;
+ }
+ for (String id : relation.getValue()) {
+ if (!ids.contains(id)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param map the map of key/value pairs in an entity
+ * @param filters the map of key/value pairs for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchFilters(Map<String, ? extends Object> map,
+ Map<String, ? extends Object> filters) {
+ for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
+ Object value = map.get(filter.getKey());
+ if (value == null) {
+ return false;
+ }
+ if (!value.equals(filter.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param entityEvents the set of event objects in an entity
+ * @param eventFilters the set of event Ids for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
+ Set<String> eventFilters) {
+ Set<String> eventIds = new HashSet<String>();
+ for (TimelineEvent event : entityEvents) {
+ eventIds.add(event.getId());
+ }
+ for (String eventFilter : eventFilters) {
+ if (!eventIds.contains(eventFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ *
+ * @param metrics the set of metric objects in an entity
+ * @param metricFilters the set of metric Ids for filtering
+ * @return a boolean flag to indicate if both match
+ */
+ public static boolean matchMetricFilters(Set<TimelineMetric> metrics,
+ Set<String> metricFilters) {
+ Set<String> metricIds = new HashSet<String>();
+ for (TimelineMetric metric : metrics) {
+ metricIds.add(metric.getId());
+ }
+
+ for (String metricFilter : metricFilters) {
+ if (!metricIds.contains(metricFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index 90da966..26e7748 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -62,7 +62,7 @@ public enum EntityColumn implements Column<EntityTable> {
private final String columnQualifier;
private final byte[] columnQualifierBytes;
- private EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ EntityColumn(ColumnFamily<EntityTable> columnFamily,
String columnQualifier) {
this.columnFamily = columnFamily;
this.columnQualifier = columnQualifier;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
index 8a95d12..7c63727 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
@@ -53,7 +53,7 @@ public enum EntityColumnFamily implements ColumnFamily<EntityTable> {
* @param value create a column family with this name. Must be lower case and
* without spaces.
*/
- private EntityColumnFamily(String value) {
+ EntityColumnFamily(String value) {
// column families should be lower case and not contain any spaces.
this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 8b7bc3e..58272ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -80,7 +80,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* @param columnFamily that this column is stored in.
* @param columnPrefix for this column.
*/
- private EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
String columnPrefix) {
column = new ColumnHelper<EntityTable>(columnFamily);
this.columnFamily = columnFamily;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 3e17ad0..9a72be0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -55,17 +55,45 @@ public class EntityRowKey {
/**
* Constructs a row key prefix for the entity table as follows:
- * {@code userName!clusterId!flowId!flowRunId!AppId}
+ * {@code userName!clusterId!flowId!flowRunId!AppId!entityType!}
*
* @param clusterId
* @param userId
* @param flowId
* @param flowRunId
* @param appId
+ * @param entityType
* @return byte array with the row key prefix
*/
+ public static byte[] getRowKeyPrefix(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId, String entityType) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
+ byte[] third =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, entityType, ""));
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Constructs a row key for the entity table as follows:
+ * {@code userName!clusterId!flowId!flowRunId!AppId!entityType!entityId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @param entityType
+ * @param entityId
+ * @return byte array with the row key
+ */
public static byte[] getRowKey(String clusterId, String userId,
- String flowId, Long flowRunId, String appId, TimelineEntity te) {
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId) {
byte[] first =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
flowId));
@@ -73,8 +101,8 @@ public class EntityRowKey {
// time.
byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third =
- Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, te.getType(),
- te.getId()));
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, entityType,
+ entityId));
return Separator.QUALIFIERS.join(first, second, third);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 2ae7d39..f657a14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.regionserver.BloomType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEntitySchemaConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
/**
* The entity table as column families info, config and metrics. Info stores
@@ -75,7 +75,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEnti
public class EntityTable extends BaseTable<EntityTable> {
/** entity prefix */
private static final String PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".entity";
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "entity";
/** config param name that specifies the entity table name */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
@@ -146,9 +146,9 @@ public class EntityTable extends BaseTable<EntityTable> {
entityTableDescp
.setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- TimelineEntitySchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
admin.createTable(entityTableDescp,
- TimelineEntitySchemaConstants.getUsernameSplits());
+ TimelineHBaseSchemaConstants.getUsernameSplits());
LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ admin.tableExists(table));
}
[34/50] [abbrv] hadoop git commit: YARN-3949. Ensure timely flush of
timeline writes. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
YARN-3949. Ensure timely flush of timeline writes. Contributed by Sangjin Lee.
(cherry picked from commit 967bef7e0396d857913caa2574afb103a5f0b81b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57f6d069
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57f6d069
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57f6d069
Branch: refs/heads/YARN-2928
Commit: 57f6d0698047c8ca9a260a497550c6656e2fd9d2
Parents: cced594
Author: Junping Du <ju...@apache.org>
Authored: Sat Jul 25 10:30:29 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:14 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++
.../src/main/resources/yarn-default.xml | 17 ++++-
.../collector/TimelineCollectorManager.java | 65 ++++++++++++++++++--
.../storage/FileSystemTimelineWriterImpl.java | 5 ++
.../storage/HBaseTimelineWriterImpl.java | 6 ++
.../storage/PhoenixTimelineWriterImpl.java | 5 ++
.../timelineservice/storage/TimelineWriter.java | 9 +++
.../TestNMTimelineCollectorManager.java | 5 ++
9 files changed, 119 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6e10926..fd19320 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -79,6 +79,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3047. [Data Serving] Set up ATS reader with basic request serving
structure and lifecycle (Varun Saxena via sjlee)
+ YARN-3949. Ensure timely flush of timeline writes. (Sangjin Lee via
+ junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2bc7d70..2c59bd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1470,6 +1470,15 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_READER_CLASS =
TIMELINE_SERVICE_PREFIX + "reader.class";
+ /** The setting that controls how often the timeline collector flushes the
+ * timeline writer.
+ */
+ public static final String TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS =
+ TIMELINE_SERVICE_PREFIX + "writer.flush-interval-seconds";
+
+ public static final int
+ DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 10450a9..8adab4d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -758,7 +758,15 @@
<name>yarn.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
-
+
+ <property>
+ <description>The setting that controls whether yarn container metrics is
+ published to the timeline server or not by RM. This configuration setting is
+ for ATS V2.</description>
+ <name>yarn.rm.system-metrics-publisher.emit-container-events</name>
+ <value>false</value>
+ </property>
+
<property>
<description>Number of worker threads that send the yarn system metrics
@@ -1880,6 +1888,13 @@
<value>${hadoop.tmp.dir}/yarn/timeline</value>
</property>
+ <property>
+ <description>The setting that controls how often the timeline collector
+ flushes the timeline writer.</description>
+ <name>yarn.timeline-service.writer.flush-interval-seconds</name>
+ <value>60</value>
+ </property>
+
<!-- Shared Cache Configuration -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 23ad4f4..e9f2085 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -19,6 +19,13 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -32,9 +39,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import com.google.common.annotations.VisibleForTesting;
/**
* Class that manages adding and removing collectors and their lifecycle. It
@@ -47,7 +52,10 @@ public abstract class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
- protected TimelineWriter writer;
+ private TimelineWriter writer;
+ private ScheduledExecutorService writerFlusher;
+ private int flushInterval;
+ private boolean writerFlusherRunning;
@Override
public void serviceInit(Configuration conf) throws Exception {
@@ -56,6 +64,12 @@ public abstract class TimelineCollectorManager extends AbstractService {
FileSystemTimelineWriterImpl.class,
TimelineWriter.class), conf);
writer.init(conf);
+ // create a single dedicated thread for flushing the writer on a periodic
+ // basis
+ writerFlusher = Executors.newSingleThreadScheduledExecutor();
+ flushInterval = conf.getInt(
+ YarnConfiguration.TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS);
super.serviceInit(conf);
}
@@ -65,6 +79,10 @@ public abstract class TimelineCollectorManager extends AbstractService {
if (writer != null) {
writer.start();
}
+ // schedule the flush task
+ writerFlusher.scheduleAtFixedRate(new WriterFlushTask(writer),
+ flushInterval, flushInterval, TimeUnit.SECONDS);
+ writerFlusherRunning = true;
}
// access to this map is synchronized with the map itself
@@ -161,9 +179,48 @@ public abstract class TimelineCollectorManager extends AbstractService {
c.serviceStop();
}
}
+ // stop the flusher first
+ if (writerFlusher != null) {
+ writerFlusher.shutdown();
+ writerFlusherRunning = false;
+ if (!writerFlusher.awaitTermination(30, TimeUnit.SECONDS)) {
+ // in reality it should be ample time for the flusher task to finish
+ // even if it times out, writers may be able to handle closing in this
+ // situation fine
+ // proceed to close the writer
+ LOG.warn("failed to stop the flusher task in time. " +
+ "will still proceed to close the writer.");
+ }
+ }
if (writer != null) {
writer.close();
}
super.serviceStop();
}
+
+ @VisibleForTesting
+ boolean writerFlusherRunning() {
+ return writerFlusherRunning;
+ }
+
+ /**
+ * Task that invokes the flush operation on the timeline writer.
+ */
+ private static class WriterFlushTask implements Runnable {
+ private final TimelineWriter writer;
+
+ public WriterFlushTask(TimelineWriter writer) {
+ this.writer = writer;
+ }
+
+ public void run() {
+ try {
+ writer.flush();
+ } catch (Throwable th) {
+ // we need to handle all exceptions or subsequent execution may be
+ // suppressed
+ LOG.error("exception during timeline writer flush!", th);
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index 34a6b7c..b22b39f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -127,6 +127,11 @@ public class FileSystemTimelineWriterImpl extends AbstractService
mkdirs(outputRoot, ENTITIES_DIR);
}
+ @Override
+ public void flush() throws IOException {
+ // no op
+ }
+
private static String mkdirs(String... dirStrs) throws IOException {
StringBuilder path = new StringBuilder();
for (String dirStr : dirStrs) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index e48ca60..876ad6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -214,6 +214,12 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
return null;
}
+ @Override
+ public void flush() throws IOException {
+ // flush all buffered mutators
+ entityTable.flush();
+ }
+
/**
* close the hbase connections The close APIs perform flushing and release any
* resources held
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
index 5b4442c..381ff17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
@@ -187,6 +187,11 @@ public class PhoenixTimelineWriterImpl extends AbstractService
}
+ @Override
+ public void flush() throws IOException {
+ // currently no-op
+ }
+
// Utility functions
@Private
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 494e8ad..50136de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -70,4 +70,13 @@ public interface TimelineWriter extends Service {
*/
TimelineWriteResponse aggregate(TimelineEntity data,
TimelineAggregationTrack track) throws IOException;
+
+ /**
+ * Flushes the data to the backend storage. Whatever may be buffered will be
+ * written to the storage when the method returns. This may be a potentially
+ * time-consuming operation, and should be used judiciously.
+ *
+ * @throws IOException
+ */
+ void flush() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57f6d069/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
index 87343fd..0d69fbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
@@ -67,6 +67,11 @@ public class TestNMTimelineCollectorManager {
}
@Test
+ public void testStartingWriterFlusher() throws Exception {
+ assertTrue(collectorManager.writerFlusherRunning());
+ }
+
+ @Test
public void testStartWebApp() throws Exception {
assertNotNull(collectorManager.getRestServerBindAddress());
String address = collectorManager.getRestServerBindAddress();
[47/50] [abbrv] hadoop git commit: YARN-3814. REST API implementation
for getting raw entities in TimelineReader (Naganarasimha G R via sjlee)
Posted by sj...@apache.org.
YARN-3814. REST API implementation for getting raw entities in TimelineReader (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/afa54d2f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/afa54d2f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/afa54d2f
Branch: refs/heads/YARN-2928
Commit: afa54d2fcc2e83dbb036a06014b3f6426555f598
Parents: e979f30
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Aug 21 19:10:23 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:52:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../reader/TimelineReaderManager.java | 41 ++
.../reader/TimelineReaderServer.java | 2 +-
.../reader/TimelineReaderWebServices.java | 245 +++++++++-
.../storage/FileSystemTimelineReaderImpl.java | 5 +
.../reader/TestTimelineReaderWebServices.java | 456 ++++++++++++++++++-
6 files changed, 741 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa54d2f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0041f7f..b1d2bd6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -97,6 +97,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4025. Deal with byte representations of Longs in writer code.
(Sangjin Lee and Vrushali C via junping_du)
+ YARN-3814. REST API implementation for getting raw entities in
+ TimelineReader (Naganarasimha G R via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa54d2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 5573185..7fafd82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -18,10 +18,18 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
@Private
@Unstable
@@ -33,4 +41,37 @@ public class TimelineReaderManager extends AbstractService {
super(TimelineReaderManager.class.getName());
this.reader = timelineReader;
}
+
+ /**
+ * Get a set of entities matching given predicates. The meaning of each
+ * argument has been documented with {@link TimelineReader#getEntities}.
+ *
+ * @see TimelineReader#getEntities
+ */
+ Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ return reader.getEntities(userId, clusterId, flowId, flowRunId, appId,
+ entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
+ modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
+ metricFilters, eventFilters, fieldsToRetrieve);
+ }
+
+ /**
+ * Get single timeline entity. The meaning of each argument has been
+ * documented with {@link TimelineReader#getEntity}.
+ *
+ * @see TimelineReader#getEntity
+ */
+ public TimelineEntity getEntity(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fields) throws IOException {
+ return reader.getEntity(userId, clusterId, flowId, flowRunId, appId,
+ entityType, entityId, fields);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa54d2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index 874112c..319cfb0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -54,7 +54,7 @@ import com.google.common.annotations.VisibleForTesting;
public class TimelineReaderServer extends CompositeService {
private static final Log LOG = LogFactory.getLog(TimelineReaderServer.class);
private static final int SHUTDOWN_HOOK_PRIORITY = 30;
- private static final String TIMELINE_READER_MANAGER_ATTR =
+ static final String TIMELINE_READER_MANAGER_ATTR =
"timeline.reader.manager";
private HttpServer2 readerWebServer;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa54d2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 3655a72..0b5fde0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -18,42 +18,283 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
+import javax.ws.rs.QueryParam;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.inject.Singleton;
-
/** REST end point for Timeline Reader */
@Private
@Unstable
@Singleton
@Path("/ws/v2/timeline")
public class TimelineReaderWebServices {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineReaderWebServices.class);
+
+ @Context private ServletContext ctxt;
+
+ private static final String COMMA_DELIMITER = ",";
+ private static final String COLON_DELIMITER = ":";
private void init(HttpServletResponse response) {
response.setContentType(null);
}
+ private static Set<String> parseValuesStr(String str, String delimiter) {
+ if (str == null || str.isEmpty()) {
+ return null;
+ }
+ Set<String> strSet = new HashSet<String>();
+ String[] strs = str.split(delimiter);
+ for (String aStr : strs) {
+ strSet.add(aStr.trim());
+ }
+ return strSet;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> void parseKeyValues(Map<String,T> map, String str,
+ String pairsDelim, String keyValuesDelim, boolean stringValue,
+ boolean multipleValues) {
+ String[] pairs = str.split(pairsDelim);
+ for (String pair : pairs) {
+ if (pair == null || pair.trim().isEmpty()) {
+ continue;
+ }
+ String[] pairStrs = pair.split(keyValuesDelim);
+ if (pairStrs.length < 2) {
+ continue;
+ }
+ if (!stringValue) {
+ try {
+ Object value =
+ GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
+ map.put(pairStrs[0].trim(), (T) value);
+ } catch (IOException e) {
+ map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
+ }
+ } else {
+ String key = pairStrs[0].trim();
+ if (multipleValues) {
+ Set<String> values = new HashSet<String>();
+ for (int i = 1; i < pairStrs.length; i++) {
+ values.add(pairStrs[i].trim());
+ }
+ map.put(key, (T) values);
+ } else {
+ map.put(key, (T) pairStrs[1].trim());
+ }
+ }
+ }
+ }
+
+ private static Map<String, Set<String>> parseKeyStrValuesStr(String str,
+ String pairsDelim, String keyValuesDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, Set<String>> map = new HashMap<String, Set<String>>();
+ parseKeyValues(map, str,pairsDelim, keyValuesDelim, true, true);
+ return map;
+ }
+
+ private static Map<String, String> parseKeyStrValueStr(String str,
+ String pairsDelim, String keyValDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, String> map = new HashMap<String, String>();
+ parseKeyValues(map, str, pairsDelim, keyValDelim, true, false);
+ return map;
+ }
+
+ private static Map<String, Object> parseKeyStrValueObj(String str,
+ String pairsDelim, String keyValDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, Object> map = new HashMap<String, Object>();
+ parseKeyValues(map, str, pairsDelim, keyValDelim, false, false);
+ return map;
+ }
+
+ private static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter);
+ EnumSet<Field> fieldList = EnumSet.noneOf(Field.class);
+ for (String s : strs) {
+ fieldList.add(Field.valueOf(s.trim().toUpperCase()));
+ }
+ return fieldList;
+ }
+
+ private static Long parseLongStr(String str) {
+ return str == null ? null : Long.parseLong(str.trim());
+ }
+
+ private static String parseStr(String str) {
+ return str == null ? null : str.trim();
+ }
+
+ private static UserGroupInformation getUser(HttpServletRequest req) {
+ String remoteUser = req.getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUGI;
+ }
+
+ private TimelineReaderManager getTimelineReaderManager() {
+ return (TimelineReaderManager)
+ ctxt.getAttribute(TimelineReaderServer.TIMELINE_READER_MANAGER_ATTR);
+ }
+
/**
* Return the description of the timeline reader web services.
*/
@GET
- @Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
+ @Produces(MediaType.APPLICATION_JSON)
public TimelineAbout about(
@Context HttpServletRequest req,
@Context HttpServletResponse res) {
init(res);
return TimelineUtils.createTimelineAbout("Timeline Reader API");
}
+
+ /**
+ * Return a set of entities that match the given parameters.
+ */
+ @GET
+ @Path("/entities/{clusterId}/{appId}/{entityType}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterId") String clusterId,
+ @PathParam("appId") String appId,
+ @PathParam("entityType") String entityType,
+ @QueryParam("userId") String userId,
+ @QueryParam("flowId") String flowId,
+ @QueryParam("flowRunId") String flowRunId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdTimeStart") String createdTimeStart,
+ @QueryParam("createdTimeEnd") String createdTimeEnd,
+ @QueryParam("modifiedTimeStart") String modifiedTimeStart,
+ @QueryParam("modifiedTimeEnd") String modifiedTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ UserGroupInformation callerUGI = getUser(req);
+ try {
+ return timelineReaderManager.getEntities(
+ callerUGI != null && (userId == null || userId.isEmpty()) ?
+ callerUGI.getUserName().trim() : parseStr(userId),
+ parseStr(clusterId), parseStr(flowId),
+ parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
+ parseLongStr(limit), parseLongStr(createdTimeStart),
+ parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart),
+ parseLongStr(modifiedTimeEnd),
+ parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
+ parseValuesStr(metricfilters, COMMA_DELIMITER),
+ parseValuesStr(eventfilters, COMMA_DELIMITER),
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException(
+ "createdTime or modifiedTime start/end or limit or flowId is not" +
+ " a numeric value.");
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Requested Invalid Field.");
+ } catch (Exception e) {
+ LOG.error("Error getting entities", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ /**
+ * Return a single entity of the given entity type and Id.
+ */
+ @GET
+ @Path("/entity/{clusterId}/{appId}/{entityType}/{entityId}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getEntity(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("clusterId") String clusterId,
+ @PathParam("appId") String appId,
+ @PathParam("entityType") String entityType,
+ @PathParam("entityId") String entityId,
+ @QueryParam("userId") String userId,
+ @QueryParam("flowId") String flowId,
+ @QueryParam("flowRunId") String flowRunId,
+ @QueryParam("fields") String fields) {
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ UserGroupInformation callerUGI = getUser(req);
+ TimelineEntity entity = null;
+ try {
+ entity = timelineReaderManager.getEntity(
+ callerUGI != null && (userId == null || userId.isEmpty()) ?
+ callerUGI.getUserName().trim() : parseStr(userId),
+ parseStr(clusterId), parseStr(flowId), parseLongStr(flowRunId),
+ parseStr(appId), parseStr(entityType), parseStr(entityId),
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (NumberFormatException e) {
+ throw new BadRequestException("flowRunId is not a numeric value.");
+ } catch (IllegalArgumentException e) {
+ throw new BadRequestException("Requested Invalid Field.");
+ } catch (Exception e) {
+ LOG.error("Error getting entity", e);
+ throw new WebApplicationException(e,
+ Response.Status.INTERNAL_SERVER_ERROR);
+ }
+ if (entity == null) {
+ throw new NotFoundException("Timeline entity {id: " + parseStr(entityId) +
+ ", type: " + parseStr(entityType) + " } is not found");
+ }
+ return entity;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa54d2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 45ddd1d..626c770 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.Charset;
@@ -397,6 +398,10 @@ public class FileSystemTimelineReaderImpl extends AbstractService
new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
return createEntityToBeReturned(entity, fieldsToRetrieve);
+ } catch (FileNotFoundException e) {
+ LOG.info("Cannot find entity {id:" + entityId + " , type:" + entityType +
+ "}. Will send HTTP 404 in response.");
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/afa54d2f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index a9145d0..0f7c22f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -18,25 +18,37 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.util.Set;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TestFileSystemTimelineReaderImpl;
import org.junit.After;
+import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
@@ -46,12 +58,23 @@ public class TestTimelineReaderWebServices {
private int serverPort;
private TimelineReaderServer server;
+ @BeforeClass
+ public static void setup() throws Exception {
+ TestFileSystemTimelineReaderImpl.setup();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ TestFileSystemTimelineReaderImpl.tearDown();
+ }
+
@Before
public void init() throws Exception {
try {
Configuration config = new YarnConfiguration();
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
"localhost:0");
+ config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
server = new TimelineReaderServer();
server.init(config);
server.start();
@@ -69,6 +92,22 @@ public class TestTimelineReaderWebServices {
}
}
+ private static TimelineEntity newEntity(String type, String id) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setIdentifier(new TimelineEntity.Identifier(type, id));
+ return entity;
+ }
+
+ private static void verifyHttpResponse(Client client, URI uri,
+ Status status) {
+ ClientResponse resp =
+ client.resource(uri).accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
+ assertNotNull(resp);
+ assertTrue("Response from server should have been " + status,
+ resp.getClientResponseStatus().equals(status));
+ }
+
private static Client createClient() {
ClientConfig cfg = new DefaultClientConfig();
cfg.getClasses().add(YarnJacksonJaxbJsonProvider.class);
@@ -76,14 +115,19 @@ public class TestTimelineReaderWebServices {
new DummyURLConnectionFactory()), cfg);
}
- private static ClientResponse getResponse(Client client, URI uri) throws Exception {
+ private static ClientResponse getResponse(Client client, URI uri)
+ throws Exception {
ClientResponse resp =
client.resource(uri).accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON).get(ClientResponse.class);
if (resp == null ||
resp.getClientResponseStatus() != ClientResponse.Status.OK) {
- System.out.println(resp.getClientResponseStatus());
- throw new IOException("Incorrect response from timeline reader.");
+ String msg = new String();
+ if (resp != null) {
+ msg = resp.getClientResponseStatus().toString();
+ }
+ throw new IOException("Incorrect response from timeline reader. " +
+ "Status=" + msg);
}
return resp;
}
@@ -102,8 +146,7 @@ public class TestTimelineReaderWebServices {
}
@Test
- public void testAbout()
- throws IOException {
+ public void testAbout() throws Exception {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/");
Client client = createClient();
try {
@@ -111,9 +154,406 @@ public class TestTimelineReaderWebServices {
TimelineAbout about = resp.getEntity(TimelineAbout.class);
Assert.assertNotNull(about);
Assert.assertEquals("Timeline Reader API", about.getAbout());
- } catch (Exception re) {
- throw new IOException(
- "Failed to get the response from timeline reader.", re);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityDefaultView() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(1425016502000L, entity.getCreatedTime());
+ assertEquals(1425016503000L, entity.getModifiedTime());
+ // Default view i.e. when no fields are specified, entity contains only
+ // entity id, entity type, created and modified time.
+ assertEquals(0, entity.getConfigs().size());
+ assertEquals(0, entity.getMetrics().size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityWithUserAndFlowInfo() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1?userId=user1&" +
+ "flowId=flow1&flowRunId=1");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(1425016502000L, entity.getCreatedTime());
+ assertEquals(1425016503000L, entity.getModifiedTime());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityCustomFields() throws Exception {
+ Client client = createClient();
+ try {
+ // Fields are case insensitive.
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1?fields=CONFIGS,Metrics,info");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(3, entity.getConfigs().size());
+ assertEquals(3, entity.getMetrics().size());
+ assertEquals(1, entity.getInfo().size());
+ // No events will be returned as events are not part of fields.
+ assertEquals(0, entity.getEvents().size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityAllFields() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_1?fields=ALL");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entity);
+ assertEquals("id_1", entity.getId());
+ assertEquals("app", entity.getType());
+ assertEquals(3, entity.getConfigs().size());
+ assertEquals(3, entity.getMetrics().size());
+ assertEquals(1, entity.getInfo().size());
+ assertEquals(2, entity.getEvents().size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntityNotPresent() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity/cluster1/app1/app/id_10");
+ verifyHttpResponse(client, uri, Status.NOT_FOUND);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntities() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(4, entities.size());
+ assertTrue("Entities id_1, id_2, id_3 and id_4 should have been" +
+ " present in response",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_2")) &&
+ entities.contains(newEntity("app", "id_3")) &&
+ entities.contains(newEntity("app", "id_4")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesWithLimit() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?limit=2");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ // Entities returned are based on most recent created time.
+ assertTrue("Entities with id_1 and id_4 should have been present " +
+ "in response based on entity created time.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?limit=3");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ // Even though 2 entities out of 4 have same created time, one entity
+ // is left out due to limit
+ assertEquals(3, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesBasedOnCreatedTime() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?createdTimeStart=1425016502030&"
+ + "createdTimeEnd=1425016502060");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_4 should have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?createdTimeEnd=1425016502010");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(3, entities.size());
+ assertFalse("Entity with id_4 should not have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?createdTimeStart=1425016502010");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_4 should have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesBasedOnModifiedTime() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?modifiedTimeStart=1425016502090"
+ + "&modifiedTimeEnd=1425016503020");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_1 and id_4 should have been" +
+ " present in response.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_4")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?modifiedTimeEnd=1425016502090");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_2 and id_3 should have been " +
+ "present in response.",
+ entities.contains(newEntity("app", "id_2")) &&
+ entities.contains(newEntity("app", "id_3")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?modifiedTimeStart=1425016503005");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_4 should have been present in response.",
+ entities.contains(newEntity("app", "id_4")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByRelations() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?relatesto=flow:flow1");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_1 should have been present in response.",
+ entities.contains(newEntity("app", "id_1")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?isrelatedto=type1:tid1_2,type2:" +
+ "tid2_1%60");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_1 should have been present in response.",
+ entities.contains(newEntity("app", "id_1")));
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app?isrelatedto=type1:tid1_1:tid1_2" +
+ ",type2:tid2_1%60");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_1 should have been present in response.",
+ entities.contains(newEntity("app", "id_1")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByConfigFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?conffilters=config_1:123," +
+ "config_3:abc");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_1 and id_3 should have been present" +
+ " in response.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_3")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByInfoFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?infofilters=info2:3.5");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_3 should have been present in response.",
+ entities.contains(newEntity("app", "id_3")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByMetricFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?metricfilters=metric3");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ assertTrue("Entities with id_1 and id_2 should have been present" +
+ " in response.",
+ entities.contains(newEntity("app", "id_1")) &&
+ entities.contains(newEntity("app", "id_2")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByEventFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?eventfilters=event_2,event_4");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_3 should have been present in response.",
+ entities.contains(newEntity("app", "id_3")));
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesNoMatch() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?metricfilters=metric7&" +
+ "isrelatedto=type1:tid1_1;tid1_2,type2:tid2_1%60&relatesto=" +
+ "flow:flow1&eventfilters=event_2,event_4&infofilters=info2:3.5" +
+ "&createdTimeStart=1425016502030&createdTimeEnd=1425016502060");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testInvalidValuesHandling() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entities/cluster1/app1/app?flowRunId=a23b");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entity/cluster1/app1/app/id_1?flowRunId=2ab15");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "entities/cluster1/app1/app/?limit=#$561av");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
} finally {
client.destroy();
}
[40/50] [abbrv] hadoop git commit: YARN-3049. [Storage
Implementation] Implement storage reader interface to fetch raw data from
HBase backend (Zhijie Shen via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bdf34f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index fd5643d..ab02779 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -38,11 +39,15 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
@@ -71,6 +76,8 @@ public class TestHBaseTimelineWriterImpl {
private static void createSchema() throws IOException {
new EntityTable()
.createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new AppToFlowTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
}
@Test
@@ -138,10 +145,15 @@ public class TestHBaseTimelineWriterImpl {
te.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
try {
Configuration c1 = util.getConfiguration();
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
String cluster = "cluster1";
String user = "user1";
String flow = "some_flow_name";
@@ -255,9 +267,22 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(1, rowCount);
assertEquals(17, colCount);
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
} finally {
- hbi.stop();
- hbi.close();
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
}
// Somewhat of a hack, not a separate test in order not to have to deal with
@@ -283,7 +308,7 @@ public class TestHBaseTimelineWriterImpl {
private void testAdditionalEntity() throws IOException {
TimelineEvent event = new TimelineEvent();
- String eventId = "foo_event_id";
+ String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
event.setId(eventId);
Long expTs = 1436512802000L;
event.setTimestamp(expTs);
@@ -291,19 +316,23 @@ public class TestHBaseTimelineWriterImpl {
Object expVal = "test";
event.addInfo(expKey, expVal);
- final TimelineEntity entity = new TimelineEntity();
- entity.setId("attempt_1329348432655_0001_m_000008_18");
- entity.setType("FOO_ATTEMPT");
+ final TimelineEntity entity = new ApplicationEntity();
+ entity.setId(ApplicationId.newInstance(0, 1).toString());
entity.addEvent(event);
TimelineEntities entities = new TimelineEntities();
entities.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
try {
Configuration c1 = util.getConfiguration();
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
String cluster = "cluster2";
String user = "user2";
String flow = "other_flow_name";
@@ -352,9 +381,31 @@ public class TestHBaseTimelineWriterImpl {
}
assertEquals(1, rowCount);
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es2 = hbr.getEntities(user, cluster, null, null,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertNotNull(e2);
+ assertEquals(e1, e2);
+ assertEquals(1, es1.size());
+ assertEquals(1, es2.size());
+ assertEquals(es1, es2);
} finally {
- hbi.stop();
- hbi.close();
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
}
}
@@ -375,10 +426,15 @@ public class TestHBaseTimelineWriterImpl {
entities.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
try {
Configuration c1 = util.getConfiguration();
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
String cluster = "cluster_emptyeventkey";
String user = "user_emptyeventkey";
String flow = "other_flow_name";
@@ -430,13 +486,21 @@ public class TestHBaseTimelineWriterImpl {
}
assertEquals(1, rowCount);
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
+ entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ appName, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
} finally {
hbi.stop();
hbi.close();
+ hbr.stop();;
+ hbr.close();
}
}
-
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
[16/50] [abbrv] hadoop git commit: MAPREDUCE-6327. Made MR AM use
timeline service v2 API to write history events and counters. Contributed by
Junping Du.
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index eab9026..b3ea26e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -18,22 +18,45 @@
package org.apache.hadoop.mapred;
+import java.io.File;
+import java.io.IOException;
+
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.jobhistory.EventType;
import org.apache.hadoop.mapreduce.jobhistory.TestJobHistoryEventHandler;
import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
+import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Assert;
import org.junit.Test;
public class TestMRTimelineEventHandling {
+ private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
+ private static final Log LOG =
+ LogFactory.getLog(TestMRTimelineEventHandling.class);
+
@Test
public void testTimelineServiceStartInMiniCluster() throws Exception {
Configuration conf = new YarnConfiguration();
@@ -47,7 +70,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestMRTimelineEventHandling.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
@@ -88,7 +111,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestMRTimelineEventHandling.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -132,6 +155,140 @@ public class TestMRTimelineEventHandling {
}
}
}
+
+ @Test
+ public void testMRNewTimelineServiceEventHandling() throws Exception {
+ LOG.info("testMRNewTimelineServiceEventHandling start.");
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+
+ // enable new timeline serivce in MR side
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED, true);
+
+ // enable aux-service based timeline collectors
+ conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
+ conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
+ + ".class", PerNodeTimelineCollectorsAuxService.class.getName());
+
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestMRTimelineEventHandling.class.getSimpleName(), 1, true);
+ cluster.init(conf);
+ cluster.start();
+ LOG.info("A MiniMRYarnCluster get start.");
+
+ Path inDir = new Path("input");
+ Path outDir = new Path("output");
+ LOG.info("Run 1st job which should be successful.");
+ RunningJob job =
+ UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
+ Assert.assertEquals(JobStatus.SUCCEEDED,
+ job.getJobStatus().getState().getValue());
+
+ YarnClient yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(new Configuration(cluster.getConfig()));
+ yarnClient.start();
+ EnumSet<YarnApplicationState> appStates =
+ EnumSet.allOf(YarnApplicationState.class);
+
+ ApplicationId firstAppId = null;
+ List<ApplicationReport> apps = yarnClient.getApplications(appStates);
+ Assert.assertEquals(apps.size(), 1);
+ ApplicationReport appReport = apps.get(0);
+ firstAppId = appReport.getApplicationId();
+
+ checkNewTimelineEvent(firstAppId);
+
+ LOG.info("Run 2nd job which should be failed.");
+ job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir);
+ Assert.assertEquals(JobStatus.FAILED,
+ job.getJobStatus().getState().getValue());
+
+ apps = yarnClient.getApplications(appStates);
+ Assert.assertEquals(apps.size(), 2);
+
+ ApplicationId secAppId = null;
+ secAppId = apps.get(0).getApplicationId() == firstAppId ?
+ apps.get(1).getApplicationId() : apps.get(0).getApplicationId();
+ checkNewTimelineEvent(firstAppId);
+
+ } finally {
+ if (cluster != null) {
+ cluster.stop();
+ }
+ // Cleanup test file
+ String testRoot =
+ FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
+ File testRootFolder = new File(testRoot);
+ if(testRootFolder.isDirectory()) {
+ FileUtils.deleteDirectory(testRootFolder);
+ }
+
+ }
+ }
+
+ private void checkNewTimelineEvent(ApplicationId appId) throws IOException {
+ String tmpRoot =
+ FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
+ + "/entities/";
+
+ File tmpRootFolder = new File(tmpRoot);
+
+ Assert.assertTrue(tmpRootFolder.isDirectory());
+ String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
+ UserGroupInformation.getCurrentUser().getShortUserName() +
+ "/" + TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+ "/1/1/" + appId.toString();
+ // for this test, we expect MAPREDUCE_JOB and MAPREDUCE_TASK dirs
+ String outputDirJob = basePath + "/MAPREDUCE_JOB/";
+
+ File entityFolder = new File(outputDirJob);
+ Assert.assertTrue("Job output directory: " + outputDirJob + " is not exist.",
+ entityFolder.isDirectory());
+
+ // check for job event file
+ String jobEventFileName = appId.toString().replaceAll("application", "job")
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String jobEventFilePath = outputDirJob + jobEventFileName;
+ File jobEventFile = new File(jobEventFilePath);
+ Assert.assertTrue("jobEventFilePath: " + jobEventFilePath + " is not exist.",
+ jobEventFile.exists());
+
+ // check for task event file
+ String outputDirTask = basePath + "/MAPREDUCE_TASK/";
+ File taskFolder = new File(outputDirTask);
+ Assert.assertTrue("Task output directory: " + outputDirTask + " is not exist.",
+ taskFolder.isDirectory());
+
+ String taskEventFileName = appId.toString().replaceAll("application", "task")
+ + "_m_000000" + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String taskEventFilePath = outputDirTask + taskEventFileName;
+ File taskEventFile = new File(taskEventFilePath);
+ Assert.assertTrue("taskEventFileName: " + taskEventFilePath + " is not exist.",
+ taskEventFile.exists());
+
+ // check for task attempt event file
+ String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/";
+ File taskAttemptFolder = new File(outputDirTaskAttempt);
+ Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt +
+ " is not exist.", taskAttemptFolder.isDirectory());
+
+ String taskAttemptEventFileName = appId.toString().replaceAll(
+ "application", "attempt") + "_m_000000_0" +
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String taskAttemptEventFilePath = outputDirTaskAttempt +
+ taskAttemptEventFileName;
+ File taskAttemptEventFile = new File(taskAttemptEventFilePath);
+ Assert.assertTrue("taskAttemptEventFileName: " + taskAttemptEventFilePath +
+ " is not exist.", taskAttemptEventFile.exists());
+ }
@Test
public void testMapreduceJobTimelineServiceEnabled()
@@ -142,7 +299,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestMRTimelineEventHandling.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e58f943/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 47b38a1..90732eb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -66,6 +66,7 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
private static final Log LOG = LogFactory.getLog(MiniMRYarnCluster.class);
private JobHistoryServer historyServer;
private JobHistoryServerWrapper historyServerWrapper;
+ private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
public MiniMRYarnCluster(String testName) {
this(testName, 1);
@@ -167,8 +168,24 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
conf.set(MRConfig.MASTER_ADDRESS, "test"); // The default is local because of
// which shuffle doesn't happen
//configure the shuffle service in NM
- conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
- new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
+ String[] nmAuxServices = conf.getStrings(YarnConfiguration.NM_AUX_SERVICES);
+ // if need to enable TIMELINE_AUX_SERVICE_NAME
+ boolean enableTimelineAuxService = false;
+ if (nmAuxServices != null) {
+ for (String nmAuxService: nmAuxServices) {
+ if (nmAuxService == TIMELINE_AUX_SERVICE_NAME) {
+ enableTimelineAuxService = true;
+ break;
+ }
+ }
+ }
+ if (enableTimelineAuxService) {
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, TIMELINE_AUX_SERVICE_NAME });
+ } else {
+ conf.setStrings(YarnConfiguration.NM_AUX_SERVICES,
+ new String[] { ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID });
+ }
conf.setClass(String.format(YarnConfiguration.NM_AUX_SERVICE_FMT,
ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID), ShuffleHandler.class,
Service.class);
[23/50] [abbrv] hadoop git commit: YARN-3411. [Storage
implementation] explore the native HBase write schema for storage (Vrushali C
via sjlee)
Posted by sj...@apache.org.
YARN-3411. [Storage implementation] explore the native HBase write schema for storage (Vrushali C via sjlee)
(cherry picked from commit 7a3068854d27eadae1c57545988f5b2029bf119a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b3edd630
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b3edd630
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b3edd630
Branch: refs/heads/YARN-2928
Commit: b3edd6301b4ec8c59928cca0d5a41c94fb23740d
Parents: e93fa60
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu May 21 14:11:01 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:12 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../collector/TimelineCollectorManager.java | 19 +
.../storage/EntityColumnDetails.java | 110 ++++++
.../storage/EntityColumnFamily.java | 95 +++++
.../storage/HBaseTimelineWriterImpl.java | 225 ++++++++++++
.../server/timelineservice/storage/Range.java | 59 ++++
.../storage/TimelineEntitySchemaConstants.java | 71 ++++
.../storage/TimelineSchemaCreator.java | 231 +++++++++++++
.../storage/TimelineWriterUtils.java | 344 +++++++++++++++++++
.../storage/TestHBaseTimelineWriterImpl.java | 292 ++++++++++++++++
10 files changed, 1448 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ec9abc9..3df01e9 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -73,6 +73,8 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3634. TestMRTimelineEventHandling and TestApplication are broken. (
Sangjin Lee via junping_du)
+ YARN-3411. [Storage implementation] explore the native HBase write schema
+ for storage (Vrushali C via sjlee)
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 953d9b7..d54715c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -59,6 +59,13 @@ public abstract class TimelineCollectorManager extends AbstractService {
super.serviceInit(conf);
}
+ @Override
+ protected void serviceStart() throws Exception {
+ super.serviceStart();
+ if (writer != null) {
+ writer.start();
+ }
+ }
// access to this map is synchronized with the map itself
private final Map<ApplicationId, TimelineCollector> collectors =
@@ -147,4 +154,16 @@ public abstract class TimelineCollectorManager extends AbstractService {
return collectors.containsKey(appId);
}
+ @Override
+ protected void serviceStop() throws Exception {
+ if (collectors != null && collectors.size() > 1) {
+ for (TimelineCollector c : collectors.values()) {
+ c.serviceStop();
+ }
+ }
+ if (writer != null) {
+ writer.close();
+ }
+ super.serviceStop();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
new file mode 100644
index 0000000..2894c41
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Contains the Info Column Family details like Column names, types and byte
+ * representations for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * object that is stored in hbase Also has utility functions for storing each of
+ * these to the backend
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+enum EntityColumnDetails {
+ ID(EntityColumnFamily.INFO, "id"),
+ TYPE(EntityColumnFamily.INFO, "type"),
+ CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
+ MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
+ FLOW_VERSION(EntityColumnFamily.INFO, "flow_version"),
+ PREFIX_IS_RELATED_TO(EntityColumnFamily.INFO, "r"),
+ PREFIX_RELATES_TO(EntityColumnFamily.INFO, "s"),
+ PREFIX_EVENTS(EntityColumnFamily.INFO, "e");
+
+ private final EntityColumnFamily columnFamily;
+ private final String value;
+ private final byte[] inBytes;
+
+ private EntityColumnDetails(EntityColumnFamily columnFamily,
+ String value) {
+ this.columnFamily = columnFamily;
+ this.value = value;
+ this.inBytes = Bytes.toBytes(this.value.toLowerCase());
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ byte[] getInBytes() {
+ return inBytes;
+ }
+
+ void store(byte[] rowKey, BufferedMutator entityTable, Object inputValue)
+ throws IOException {
+ TimelineWriterUtils.store(rowKey, entityTable,
+ this.columnFamily.getInBytes(), null, this.getInBytes(), inputValue,
+ null);
+ }
+
+ /**
+ * stores events data with column prefix
+ */
+ void store(byte[] rowKey, BufferedMutator entityTable, byte[] idBytes,
+ String key, Object inputValue) throws IOException {
+ TimelineWriterUtils.store(rowKey, entityTable,
+ this.columnFamily.getInBytes(),
+ // column prefix
+ TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
+ this.getInBytes(), idBytes),
+ // column qualifier
+ Bytes.toBytes(key),
+ inputValue, null);
+ }
+
+ /**
+ * stores relation entities with a column prefix
+ */
+ void store(byte[] rowKey, BufferedMutator entityTable, String key,
+ Set<String> inputValue) throws IOException {
+ TimelineWriterUtils.store(rowKey, entityTable,
+ this.columnFamily.getInBytes(),
+ // column prefix
+ this.getInBytes(),
+ // column qualifier
+ Bytes.toBytes(key),
+ // value
+ TimelineWriterUtils.getValueAsString(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, inputValue),
+ // cell timestamp
+ null);
+ }
+
+ // TODO add a method that accepts a byte array,
+ // iterates over the enum and returns an enum from those bytes
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
new file mode 100644
index 0000000..e556351
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Contains the Column family names and byte representations for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * object that is stored in hbase
+ * Also has utility functions for storing each of these to the backend
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+enum EntityColumnFamily {
+ INFO("i"),
+ CONFIG("c"),
+ METRICS("m");
+
+ private final String value;
+ private final byte[] inBytes;
+
+ private EntityColumnFamily(String value) {
+ this.value = value;
+ this.inBytes = Bytes.toBytes(this.value.toLowerCase());
+ }
+
+ byte[] getInBytes() {
+ return inBytes;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ /**
+ * stores the key as column and value as hbase column value in the given
+ * column family in the entity table
+ *
+ * @param rowKey
+ * @param entityTable
+ * @param inputValue
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, BufferedMutator entityTable, String key,
+ String inputValue) throws IOException {
+ if (key == null) {
+ return;
+ }
+ TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
+ Bytes.toBytes(key), inputValue, null);
+ }
+
+ /**
+ * stores the values along with cell timestamp
+ *
+ * @param rowKey
+ * @param entityTable
+ * @param key
+ * @param timestamp
+ * @param inputValue
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, BufferedMutator entityTable, String key,
+ Long timestamp, Number inputValue) throws IOException {
+ if (key == null) {
+ return;
+ }
+ TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
+ Bytes.toBytes(key), inputValue, timestamp);
+ }
+
+ // TODO add a method that accepts a byte array,
+ // iterates over the enum and returns an enum from those bytes
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
new file mode 100644
index 0000000..aa71c6c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -0,0 +1,225 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineEntitySchemaConstants;
+
+/**
+ * This implements a hbase based backend for storing application timeline entity
+ * information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class HBaseTimelineWriterImpl extends AbstractService implements
+ TimelineWriter {
+
+ private Connection conn;
+ private BufferedMutator entityTable;
+
+ private static final Log LOG = LogFactory
+ .getLog(HBaseTimelineWriterImpl.class);
+
+ public HBaseTimelineWriterImpl() {
+ super(HBaseTimelineWriterImpl.class.getName());
+ }
+
+ public HBaseTimelineWriterImpl(Configuration conf) throws IOException {
+ super(conf.get("yarn.application.id",
+ HBaseTimelineWriterImpl.class.getName()));
+ }
+
+ /**
+ * initializes the hbase connection to write to the entity table
+ */
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ Configuration hbaseConf = HBaseConfiguration.create(conf);
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ TableName entityTableName = TableName.valueOf(hbaseConf.get(
+ TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
+ TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
+ entityTable = conn.getBufferedMutator(entityTableName);
+ }
+
+ /**
+ * Stores the entire information in TimelineEntities to the timeline store.
+ */
+ @Override
+ public TimelineWriteResponse write(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntities data) throws IOException {
+
+ byte[] rowKeyPrefix = TimelineWriterUtils.getRowKeyPrefix(clusterId,
+ userId, flowName, flowRunId, appId);
+
+ TimelineWriteResponse putStatus = new TimelineWriteResponse();
+ for (TimelineEntity te : data.getEntities()) {
+
+ // a set can have at most 1 null
+ if (te == null) {
+ continue;
+ }
+ // get row key
+ byte[] row = TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, rowKeyPrefix,
+ Bytes.toBytes(te.getType()), Bytes.toBytes(te.getId()));
+
+ storeInfo(row, te, flowVersion);
+ storeEvents(row, te.getEvents());
+ storeConfig(row, te.getConfigs());
+ storeMetrics(row, te.getMetrics());
+ storeRelations(row, te.getIsRelatedToEntities(),
+ EntityColumnDetails.PREFIX_IS_RELATED_TO);
+ storeRelations(row, te.getRelatesToEntities(),
+ EntityColumnDetails.PREFIX_RELATES_TO);
+ }
+
+ return putStatus;
+ }
+
+ /**
+ * Stores the Relations from the {@linkplain TimelineEntity} object
+ */
+ private void storeRelations(byte[] rowKey,
+ Map<String, Set<String>> connectedEntities,
+ EntityColumnDetails columnNamePrefix) throws IOException {
+ for (Map.Entry<String, Set<String>> entry : connectedEntities.entrySet()) {
+ columnNamePrefix.store(rowKey, entityTable, entry.getKey(),
+ entry.getValue());
+ }
+ }
+
+ /**
+ * Stores information from the {@linkplain TimelineEntity} object
+ */
+ private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion)
+ throws IOException {
+
+ EntityColumnDetails.ID.store(rowKey, entityTable, te.getId());
+ EntityColumnDetails.TYPE.store(rowKey, entityTable, te.getType());
+ EntityColumnDetails.CREATED_TIME.store(rowKey, entityTable,
+ te.getCreatedTime());
+ EntityColumnDetails.MODIFIED_TIME.store(rowKey, entityTable,
+ te.getModifiedTime());
+ EntityColumnDetails.FLOW_VERSION.store(rowKey, entityTable, flowVersion);
+ }
+
+ /**
+ * stores the config information from {@linkplain TimelineEntity}
+ */
+ private void storeConfig(byte[] rowKey, Map<String, String> config)
+ throws IOException {
+ if (config == null) {
+ return;
+ }
+ for (Map.Entry<String, String> entry : config.entrySet()) {
+ EntityColumnFamily.CONFIG.store(rowKey, entityTable,
+ entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * stores the {@linkplain TimelineMetric} information from the
+ * {@linkplain TimelineEvent} object
+ */
+ private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics)
+ throws IOException {
+ if (metrics != null) {
+ for (TimelineMetric metric : metrics) {
+ String key = metric.getId();
+ Map<Long, Number> timeseries = metric.getValues();
+ for (Map.Entry<Long, Number> entry : timeseries.entrySet()) {
+ EntityColumnFamily.METRICS.store(rowKey, entityTable, key,
+ entry.getKey(), entry.getValue());
+ }
+ }
+ }
+ }
+
+ /**
+ * Stores the events from the {@linkplain TimelineEvent} object
+ */
+ private void storeEvents(byte[] rowKey, Set<TimelineEvent> events)
+ throws IOException {
+ if (events != null) {
+ for (TimelineEvent event : events) {
+ if (event != null) {
+ String id = event.getId();
+ if (id != null) {
+ byte[] idBytes = Bytes.toBytes(id);
+ Map<String, Object> eventInfo = event.getInfo();
+ if (eventInfo != null) {
+ for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
+ EntityColumnDetails.PREFIX_EVENTS.store(rowKey,
+ entityTable, idBytes, info.getKey(), info.getValue());
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public TimelineWriteResponse aggregate(TimelineEntity data,
+ TimelineAggregationTrack track) throws IOException {
+ return null;
+ }
+
+ /**
+ * close the hbase connections
+ * The close APIs perform flushing and release any
+ * resources held
+ */
+ @Override
+ protected void serviceStop() throws Exception {
+ if (entityTable != null) {
+ LOG.info("closing entity table");
+ // The close API performs flushing and releases any resources held
+ entityTable.close();
+ }
+ if (conn != null) {
+ LOG.info("closing the hbase Connection");
+ conn.close();
+ }
+ super.serviceStop();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
new file mode 100644
index 0000000..2a2db81
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Range {
+ private final int startIdx;
+ private final int endIdx;
+
+ /**
+ * Defines a range from start index (inclusive) to end index (exclusive).
+ *
+ * @param start
+ * Starting index position
+ * @param end
+ * Ending index position (exclusive)
+ */
+ public Range(int start, int end) {
+ if (start < 0 || end < start) {
+ throw new IllegalArgumentException(
+ "Invalid range, required that: 0 <= start <= end; start=" + start
+ + ", end=" + end);
+ }
+
+ this.startIdx = start;
+ this.endIdx = end;
+ }
+
+ public int start() {
+ return startIdx;
+ }
+
+ public int end() {
+ return endIdx;
+ }
+
+ public int length() {
+ return endIdx - startIdx;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
new file mode 100644
index 0000000..d95cbb2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineEntitySchemaConstants {
+
+ /** entity prefix */
+ public static final String ENTITY_PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX
+ + ".entity";
+
+ /** config param name that specifies the entity table name */
+ public static final String ENTITY_TABLE_NAME = ENTITY_PREFIX
+ + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * entity table
+ */
+ public static final String ENTITY_TABLE_METRICS_TTL = ENTITY_PREFIX
+ + ".table.metrics.ttl";
+
+ /** default value for entity table name */
+ public static final String DEFAULT_ENTITY_TABLE_NAME = "timelineservice.entity";
+
+ /** in bytes default value for entity table name */
+ static final byte[] DEFAULT_ENTITY_TABLE_NAME_BYTES = Bytes
+ .toBytes(DEFAULT_ENTITY_TABLE_NAME);
+
+ /** separator in row key */
+ public static final String ROW_KEY_SEPARATOR = "!";
+
+ /** byte representation of the separator in row key */
+ static final byte[] ROW_KEY_SEPARATOR_BYTES = Bytes
+ .toBytes(ROW_KEY_SEPARATOR);
+
+ public static final byte ZERO_BYTES = 0;
+
+ /** default TTL is 30 days for metrics timeseries */
+ public static final int ENTITY_TABLE_METRICS_TTL_DEFAULT = 2592000;
+
+ /** default max number of versions */
+ public static final int ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT = 1000;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
new file mode 100644
index 0000000..820a6d1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -0,0 +1,231 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.hadoop.util.GenericOptionsParser;
+
+/**
+ * This creates the schema for a hbase based backend for storing application
+ * timeline information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineSchemaCreator {
+
+ final static String NAME = TimelineSchemaCreator.class.getSimpleName();
+ private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
+ final static byte[][] splits = { Bytes.toBytes("a"), Bytes.toBytes("ad"),
+ Bytes.toBytes("an"), Bytes.toBytes("b"), Bytes.toBytes("ca"),
+ Bytes.toBytes("cl"), Bytes.toBytes("d"), Bytes.toBytes("e"),
+ Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
+ Bytes.toBytes("i"), Bytes.toBytes("j"), Bytes.toBytes("k"),
+ Bytes.toBytes("l"), Bytes.toBytes("m"), Bytes.toBytes("n"),
+ Bytes.toBytes("o"), Bytes.toBytes("q"), Bytes.toBytes("r"),
+ Bytes.toBytes("s"), Bytes.toBytes("se"), Bytes.toBytes("t"),
+ Bytes.toBytes("u"), Bytes.toBytes("v"), Bytes.toBytes("w"),
+ Bytes.toBytes("x"), Bytes.toBytes("y"), Bytes.toBytes("z") };
+
+ public static final String SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ public static void main(String[] args) throws Exception {
+
+ Configuration hbaseConf = HBaseConfiguration.create();
+ // Grab input args and allow for -Dxyz style arguments
+ String[] otherArgs = new GenericOptionsParser(hbaseConf, args)
+ .getRemainingArgs();
+
+ // Grab the arguments we're looking for.
+ CommandLine commandLine = parseArgs(otherArgs);
+
+ // Grab the entityTableName argument
+ String entityTableName = commandLine.getOptionValue("e");
+ if (StringUtils.isNotBlank(entityTableName)) {
+ hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
+ entityTableName);
+ }
+ String entityTable_TTL_Metrics = commandLine.getOptionValue("m");
+ if (StringUtils.isNotBlank(entityTable_TTL_Metrics)) {
+ hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
+ entityTable_TTL_Metrics);
+ }
+ createAllTables(hbaseConf);
+ }
+
+ /**
+ * Parse command-line arguments.
+ *
+ * @param args
+ * command line arguments passed to program.
+ * @return parsed command line.
+ * @throws ParseException
+ */
+ private static CommandLine parseArgs(String[] args) throws ParseException {
+ Options options = new Options();
+
+ // Input
+ Option o = new Option("e", "entityTableName", true, "entity table name");
+ o.setArgName("entityTableName");
+ o.setRequired(false);
+ options.addOption(o);
+
+ o = new Option("m", "metricsTTL", true, "TTL for metrics column family");
+ o.setArgName("metricsTTL");
+ o.setRequired(false);
+ options.addOption(o);
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine commandLine = null;
+ try {
+ commandLine = parser.parse(options, args);
+ } catch (Exception e) {
+ LOG.error("ERROR: " + e.getMessage() + "\n");
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(NAME + " ", options, true);
+ System.exit(-1);
+ }
+
+ return commandLine;
+ }
+
+ private static void createAllTables(Configuration hbaseConf)
+ throws IOException {
+
+ Connection conn = null;
+ try {
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ Admin admin = conn.getAdmin();
+ if (admin == null) {
+ throw new IOException("Cannot create table since admin is null");
+ }
+ createTimelineEntityTable(admin, hbaseConf);
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ /**
+ * Creates a table with column families info, config and metrics
+ * info stores information about a timeline entity object
+ * config stores configuration data of a timeline entity object
+ * metrics stores the metrics of a timeline entity object
+ *
+ * Example entity table record:
+ * <pre>
+ *|------------------------------------------------------------|
+ *| Row | Column Family | Column Family | Column Family|
+ *| key | info | metrics | config |
+ *|------------------------------------------------------------|
+ *| userName! | id:entityId | metricName1: | configKey1: |
+ *| clusterId! | | metricValue1 | configValue1 |
+ *| flowId! | type:entityType| @timestamp1 | |
+ *| flowRunId! | | | configKey2: |
+ *| AppId! | created_time: | metricName1: | configValue2 |
+ *| entityType!| 1392993084018 | metricValue2 | |
+ *| entityId | | @timestamp2 | |
+ *| | modified_time: | | |
+ *| | 1392995081012 | metricName2: | |
+ *| | | metricValue1 | |
+ *| | r!relatesToKey:| @timestamp2 | |
+ *| | id3!id4!id5 | | |
+ *| | | | |
+ *| | s!isRelatedToKey| | |
+ *| | id7!id9!id5 | | |
+ *| | | | |
+ *| | e!eventKey: | | |
+ *| | eventValue | | |
+ *| | | | |
+ *| | flowVersion: | | |
+ *| | versionValue | | |
+ *|------------------------------------------------------------|
+ *</pre>
+ * @param admin
+ * @param hbaseConf
+ * @throws IOException
+ */
+ public static void createTimelineEntityTable(Admin admin,
+ Configuration hbaseConf) throws IOException {
+
+ TableName table = TableName.valueOf(hbaseConf.get(
+ TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
+ TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor entityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor cf1 = new HColumnDescriptor(
+ EntityColumnFamily.INFO.getInBytes());
+ cf1.setBloomFilterType(BloomType.ROWCOL);
+ entityTableDescp.addFamily(cf1);
+
+ HColumnDescriptor cf2 = new HColumnDescriptor(
+ EntityColumnFamily.CONFIG.getInBytes());
+ cf2.setBloomFilterType(BloomType.ROWCOL);
+ cf2.setBlockCacheEnabled(true);
+ entityTableDescp.addFamily(cf2);
+
+ HColumnDescriptor cf3 = new HColumnDescriptor(
+ EntityColumnFamily.METRICS.getInBytes());
+ entityTableDescp.addFamily(cf3);
+ cf3.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ cf3.setMinVersions(1);
+ cf3.setMaxVersions(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT);
+ cf3.setTimeToLive(hbaseConf.getInt(
+ TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
+ TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL_DEFAULT));
+ entityTableDescp
+ .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(entityTableDescp, splits);
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
new file mode 100644
index 0000000..113935e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
@@ -0,0 +1,344 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.Range;
+
+/**
+ * bunch of utility functions used across TimelineWriter classes
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineWriterUtils {
+
+ /** empty bytes */
+ public static final byte[] EMPTY_BYTES = new byte[0];
+ private static final String SPACE = " ";
+ private static final String UNDERSCORE = "_";
+ private static final String EMPTY_STRING = "";
+
+ /**
+ * Returns a single byte array containing all of the individual component
+ * arrays separated by the separator array.
+ *
+ * @param separator
+ * @param components
+ * @return byte array after joining the components
+ */
+ public static byte[] join(byte[] separator, byte[]... components) {
+ if (components == null || components.length == 0) {
+ return EMPTY_BYTES;
+ }
+
+ int finalSize = 0;
+ if (separator != null) {
+ finalSize = separator.length * (components.length - 1);
+ }
+ for (byte[] comp : components) {
+ if (comp != null) {
+ finalSize += comp.length;
+ }
+ }
+
+ byte[] buf = new byte[finalSize];
+ int offset = 0;
+ for (int i = 0; i < components.length; i++) {
+ if (components[i] != null) {
+ System.arraycopy(components[i], 0, buf, offset, components[i].length);
+ offset += components[i].length;
+ if (i < (components.length - 1) && separator != null
+ && separator.length > 0) {
+ System.arraycopy(separator, 0, buf, offset, separator.length);
+ offset += separator.length;
+ }
+ }
+ }
+ return buf;
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @return byte[] array after splitting the source
+ */
+ public static byte[][] split(byte[] source, byte[] separator) {
+ return split(source, separator, -1);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @param limit
+ * @return byte[][] after splitting the input source
+ */
+ public static byte[][] split(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = splitRanges(source, separator, limit);
+
+ byte[][] splits = new byte[segments.size()][];
+ for (int i = 0; i < segments.size(); i++) {
+ Range r = segments.get(i);
+ byte[] tmp = new byte[r.length()];
+ if (tmp.length > 0) {
+ System.arraycopy(source, r.start(), tmp, 0, r.length());
+ }
+ splits[i] = tmp;
+ }
+ return splits;
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator) {
+ return splitRanges(source, separator, -1);
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ * @param source the source data
+ * @param separator the separator pattern to look for
+ * @param limit the maximum number of splits to identify in the source
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = new ArrayList<Range>();
+ if ((source == null) || (separator == null)) {
+ return segments;
+ }
+ int start = 0;
+ itersource: for (int i = 0; i < source.length; i++) {
+ for (int j = 0; j < separator.length; j++) {
+ if (source[i + j] != separator[j]) {
+ continue itersource;
+ }
+ }
+ // all separator elements matched
+ if (limit > 0 && segments.size() >= (limit-1)) {
+ // everything else goes in one final segment
+ break;
+ }
+
+ segments.add(new Range(start, i));
+ start = i + separator.length;
+ // i will be incremented again in outer for loop
+ i += separator.length-1;
+ }
+ // add in remaining to a final range
+ if (start <= source.length) {
+ segments.add(new Range(start, source.length));
+ }
+ return segments;
+ }
+
+ /**
+ * converts run id into it's inverse timestamp
+ * @param flowRunId
+ * @return inverted long
+ */
+ public static long encodeRunId(Long flowRunId) {
+ return Long.MAX_VALUE - flowRunId;
+ }
+
+ /**
+ * return a value from the Map as a String
+ * @param key
+ * @param values
+ * @return value as a String or ""
+ * @throws IOException
+ */
+ public static String getValueAsString(final byte[] key,
+ final Map<byte[], byte[]> values) throws IOException {
+ if( values == null ) {
+ return EMPTY_STRING;
+ }
+ byte[] value = values.get(key);
+ if (value != null) {
+ return GenericObjectMapper.read(value).toString();
+ } else {
+ return EMPTY_STRING;
+ }
+ }
+
+ /**
+ * return a value from the Map as a long
+ * @param key
+ * @param values
+ * @return value as Long or 0L
+ * @throws IOException
+ */
+ public static long getValueAsLong(final byte[] key,
+ final Map<byte[], byte[]> values) throws IOException {
+ if (values == null) {
+ return 0;
+ }
+ byte[] value = values.get(key);
+ if (value != null) {
+ Number val = (Number) GenericObjectMapper.read(value);
+ return val.longValue();
+ } else {
+ return 0L;
+ }
+ }
+
+ /**
+ * concates the values from a Set<Strings> to return a single delimited string value
+ * @param rowKeySeparator
+ * @param values
+ * @return Value from the set of strings as a string
+ */
+ public static String getValueAsString(String rowKeySeparator,
+ Set<String> values) {
+
+ if (values == null) {
+ return EMPTY_STRING;
+ }
+ StringBuilder concatStrings = new StringBuilder();
+ for (String value : values) {
+ concatStrings.append(value);
+ concatStrings.append(rowKeySeparator);
+ }
+ // remove the last separator
+ if(concatStrings.length() > 1) {
+ concatStrings.deleteCharAt(concatStrings.lastIndexOf(rowKeySeparator));
+ }
+ return concatStrings.toString();
+ }
+ /**
+ * Constructs a row key prefix for the entity table
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key prefix
+ */
+ static byte[] getRowKeyPrefix(String clusterId, String userId, String flowId,
+ Long flowRunId, String appId) {
+ return TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
+ Bytes.toBytes(cleanse(userId)), Bytes.toBytes(cleanse(clusterId)),
+ Bytes.toBytes(cleanse(flowId)),
+ Bytes.toBytes(TimelineWriterUtils.encodeRunId(flowRunId)),
+ Bytes.toBytes(cleanse(appId)));
+ }
+
+ /**
+ * Takes a string token to be used as a key or qualifier and
+ * cleanses out reserved tokens.
+ * This operation is not symmetrical.
+ * Logic is to replace all spaces and separator chars in input with
+ * underscores.
+ *
+ * @param token token to cleanse.
+ * @return String with no spaces and no separator chars
+ */
+ public static String cleanse(String token) {
+ if (token == null || token.length() == 0) {
+ return token;
+ }
+
+ String cleansed = token.replaceAll(SPACE, UNDERSCORE);
+ cleansed = cleansed.replaceAll(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, UNDERSCORE);
+
+ return cleansed;
+ }
+
+ /**
+ * stores the info to the table in hbase
+ *
+ * @param rowKey
+ * @param table
+ * @param columnFamily
+ * @param columnPrefix
+ * @param columnQualifier
+ * @param inputValue
+ * @param cellTimeStamp
+ * @throws IOException
+ */
+ public static void store(byte[] rowKey, BufferedMutator table, byte[] columnFamily,
+ byte[] columnPrefix, byte[] columnQualifier, Object inputValue,
+ Long cellTimeStamp) throws IOException {
+ if ((rowKey == null) || (table == null) || (columnFamily == null)
+ || (columnQualifier == null) || (inputValue == null)) {
+ return;
+ }
+
+ Put p = null;
+ if (cellTimeStamp == null) {
+ if (columnPrefix != null) {
+ // store with prefix
+ p = new Put(rowKey);
+ p.addColumn(
+ columnFamily,
+ join(TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
+ columnPrefix, columnQualifier), GenericObjectMapper
+ .write(inputValue));
+ } else {
+ // store without prefix
+ p = new Put(rowKey);
+ p.addColumn(columnFamily, columnQualifier,
+ GenericObjectMapper.write(inputValue));
+ }
+ } else {
+ // store with cell timestamp
+ Cell cell = CellUtil.createCell(rowKey, columnFamily, columnQualifier,
+ // set the cell timestamp
+ cellTimeStamp,
+ // KeyValue Type minimum
+ TimelineEntitySchemaConstants.ZERO_BYTES,
+ GenericObjectMapper.write(inputValue));
+ p = new Put(rowKey);
+ p.add(cell);
+ }
+ if (p != null) {
+ table.mutate(p);
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b3edd630/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
new file mode 100644
index 0000000..48bacd6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -0,0 +1,292 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotNull;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.junit.BeforeClass;
+import org.junit.AfterClass;
+import org.junit.Test;
+
+/**
+ * Unit test HBaseTimelineWriterImpl
+ * YARN 3411
+ *
+ * @throws Exception
+ */
+public class TestHBaseTimelineWriterImpl {
+
+ private static HBaseTestingUtility util;
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ byte[][] families = new byte[3][];
+ families[0] = EntityColumnFamily.INFO.getInBytes();
+ families[1] = EntityColumnFamily.CONFIG.getInBytes();
+ families[2] = EntityColumnFamily.METRICS.getInBytes();
+ TimelineSchemaCreator.createTimelineEntityTable(util.getHBaseAdmin(),
+ util.getConfiguration());
+ }
+
+ @Test
+ public void testWriteEntityToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello";
+ String type = "world";
+ entity.setId(id);
+ entity.setType(type);
+ Long cTime = 1425016501000L;
+ Long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ metricValues.put(1429741609000L, 100000000);
+ metricValues.put(1429742609000L, 200000000);
+ metricValues.put(1429743609000L, 300000000);
+ metricValues.put(1429744609000L, 400000000);
+ metricValues.put(1429745609000L, 50000000000L);
+ metricValues.put(1429746609000L, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String cluster = "cluster1";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.stop();
+
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ byte[] startRow = TimelineWriterUtils.getRowKeyPrefix(cluster, user, flow,
+ runid, appName);
+ s.setStartRow(startRow);
+ s.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner = null;
+ TableName entityTableName = TableName
+ .valueOf(TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Table entityTable = conn.getTable(entityTableName);
+ int rowCount = 0;
+ int colCount = 0;
+ scanner = entityTable.getScanner(s);
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+ colCount += result.size();
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ // check info column family
+ NavigableMap<byte[], byte[]> infoValues = result
+ .getFamilyMap(EntityColumnFamily.INFO.getInBytes());
+ String id1 = TimelineWriterUtils.getValueAsString(
+ EntityColumnDetails.ID.getInBytes(), infoValues);
+ assertEquals(id, id1);
+ String type1 = TimelineWriterUtils.getValueAsString(
+ EntityColumnDetails.TYPE.getInBytes(), infoValues);
+ assertEquals(type, type1);
+ Long cTime1 = TimelineWriterUtils.getValueAsLong(
+ EntityColumnDetails.CREATED_TIME.getInBytes(), infoValues);
+ assertEquals(cTime1, cTime);
+ Long mTime1 = TimelineWriterUtils.getValueAsLong(
+ EntityColumnDetails.MODIFIED_TIME.getInBytes(), infoValues);
+ assertEquals(mTime1, mTime);
+ checkRelatedEntities(isRelatedTo, infoValues,
+ EntityColumnDetails.PREFIX_IS_RELATED_TO.getInBytes());
+ checkRelatedEntities(relatesTo, infoValues,
+ EntityColumnDetails.PREFIX_RELATES_TO.getInBytes());
+
+ // check config column family
+ NavigableMap<byte[], byte[]> configValuesResult = result
+ .getFamilyMap(EntityColumnFamily.CONFIG.getInBytes());
+ checkConfigs(configValuesResult, conf);
+
+ NavigableMap<byte[], byte[]> metricsResult = result
+ .getFamilyMap(EntityColumnFamily.METRICS.getInBytes());
+ checkMetricsSizeAndKey(metricsResult, metrics);
+ List<Cell> metricCells = result.getColumnCells(
+ EntityColumnFamily.METRICS.getInBytes(),
+ Bytes.toBytes(m1.getId()));
+ checkMetricsTimeseries(metricCells, m1);
+ }
+ }
+ assertEquals(1, rowCount);
+ assertEquals(15, colCount);
+
+ } finally {
+ hbi.stop();
+ hbi.close();
+ }
+ }
+
+ private void checkMetricsTimeseries(List<Cell> metricCells,
+ TimelineMetric m1) throws IOException {
+ Map<Long, Number> timeseries = m1.getValues();
+ assertEquals(metricCells.size(), timeseries.size());
+ for (Cell c1 : metricCells) {
+ assertTrue(timeseries.containsKey(c1.getTimestamp()));
+ assertEquals(GenericObjectMapper.read(CellUtil.cloneValue(c1)),
+ timeseries.get(c1.getTimestamp()));
+ }
+ }
+
+ private void checkMetricsSizeAndKey(
+ NavigableMap<byte[], byte[]> metricsResult, Set<TimelineMetric> metrics) {
+ assertEquals(metrics.size(), metricsResult.size());
+ for (TimelineMetric m1 : metrics) {
+ byte[] key = Bytes.toBytes(m1.getId());
+ assertTrue(metricsResult.containsKey(key));
+ }
+ }
+
+ private void checkConfigs(NavigableMap<byte[], byte[]> configValuesResult,
+ Map<String, String> conf) throws IOException {
+
+ assertEquals(conf.size(), configValuesResult.size());
+ byte[] columnName;
+ for (String key : conf.keySet()) {
+ columnName = Bytes.toBytes(key);
+ assertTrue(configValuesResult.containsKey(columnName));
+ byte[] value = configValuesResult.get(columnName);
+ assertNotNull(value);
+ assertEquals(conf.get(key), GenericObjectMapper.read(value));
+ }
+ }
+
+ private void checkRelatedEntities(Map<String, Set<String>> isRelatedTo,
+ NavigableMap<byte[], byte[]> infoValues, byte[] columnPrefix)
+ throws IOException {
+
+ for (String key : isRelatedTo.keySet()) {
+ byte[] columnName = TimelineWriterUtils.join(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, columnPrefix,
+ Bytes.toBytes(key));
+
+ byte[] value = infoValues.get(columnName);
+ assertNotNull(value);
+ String isRelatedToEntities = GenericObjectMapper.read(value).toString();
+ assertNotNull(isRelatedToEntities);
+ assertEquals(
+ TimelineWriterUtils.getValueAsString(
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR,
+ isRelatedTo.get(key)), isRelatedToEntities);
+ }
+ }
+
+ private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
+ String flow, Long runid, String appName, TimelineEntity te) {
+
+ byte[][] rowKeyComponents = TimelineWriterUtils.split(rowKey,
+ TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES);
+
+ assertTrue(rowKeyComponents.length == 7);
+ assertEquals(user, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.encodeRunId(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(TimelineWriterUtils.cleanse(appName), Bytes.toString(rowKeyComponents[4]));
+ assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
+ assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
+ return true;
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
[37/50] [abbrv] hadoop git commit: YARN-3906. Split the application
table from the entity table. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index ab02779..95f88d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -47,6 +48,10 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
@@ -60,7 +65,15 @@ import org.junit.BeforeClass;
import org.junit.Test;
/**
- * @throws Exception
+ * Various tests to test writing entities to HBase and reading them back from
+ * it.
+ *
+ * It uses a single HBase mini-cluster for all tests which is a little more
+ * realistic, and helps test correctness in the presence of other data.
+ *
+ * Each test uses a different cluster name to be able to handle its own data
+ * even if other records exist in the table. Use a different cluster name if
+ * you add a new test.
*/
public class TestHBaseTimelineWriterImpl {
@@ -78,6 +91,199 @@ public class TestHBaseTimelineWriterImpl {
.createTable(util.getHBaseAdmin(), util.getConfiguration());
new AppToFlowTable()
.createTable(util.getHBaseAdmin(), util.getConfiguration());
+ new ApplicationTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
+ }
+
+ @Test
+ public void testWriteApplicationToHBase() throws Exception {
+ TimelineEntities te = new TimelineEntities();
+ ApplicationEntity entity = new ApplicationEntity();
+ String id = "hello";
+ entity.setId(id);
+ Long cTime = 1425016501000L;
+ Long mTime = 1425026901000L;
+ entity.setCreatedTime(cTime);
+ entity.setModifiedTime(mTime);
+
+ // add the info map in Timeline Entity
+ Map<String, Object> infoMap = new HashMap<String, Object>();
+ infoMap.put("infoMapKey1", "infoMapValue1");
+ infoMap.put("infoMapKey2", 10);
+ entity.addInfo(infoMap);
+
+ // add the isRelatedToEntity info
+ String key = "task";
+ String value = "is_related_to_entity_id_here";
+ Set<String> isRelatedToSet = new HashSet<String>();
+ isRelatedToSet.add(value);
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ isRelatedTo.put(key, isRelatedToSet);
+ entity.setIsRelatedToEntities(isRelatedTo);
+
+ // add the relatesTo info
+ key = "container";
+ value = "relates_to_entity_id_here";
+ Set<String> relatesToSet = new HashSet<String>();
+ relatesToSet.add(value);
+ value = "relates_to_entity_id_here_Second";
+ relatesToSet.add(value);
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ relatesTo.put(key, relatesToSet);
+ entity.setRelatesToEntities(relatesTo);
+
+ // add some config entries
+ Map<String, String> conf = new HashMap<String, String>();
+ conf.put("config_param1", "value1");
+ conf.put("config_param2", "value2");
+ entity.addConfigs(conf);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = System.currentTimeMillis();
+ metricValues.put(ts - 120000, 100000000);
+ metricValues.put(ts - 100000, 200000000);
+ metricValues.put(ts - 80000, 300000000);
+ metricValues.put(ts - 60000, 400000000);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity.addMetrics(metrics);
+
+ te.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.start();
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+ String cluster = "cluster_test_write_app";
+ String user = "user1";
+ String flow = "some_flow_name";
+ String flowVersion = "AB7822C10F1111";
+ long runid = 1002345678919L;
+ hbi.write(cluster, user, flow, flowVersion, runid, id, te);
+ hbi.stop();
+
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, id);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+ assertEquals(16, result.size());
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ id));
+
+ // check info column family
+ String id1 = ApplicationColumn.ID.readResult(result).toString();
+ assertEquals(id, id1);
+
+ Number val =
+ (Number) ApplicationColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = val.longValue();
+ assertEquals(cTime1, cTime);
+
+ val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
+ Long mTime1 = val.longValue();
+ assertEquals(mTime1, mTime);
+
+ Map<String, Object> infoColumns =
+ ApplicationColumnPrefix.INFO.readResults(result);
+ assertEquals(infoMap.size(), infoColumns.size());
+ for (String infoItem : infoMap.keySet()) {
+ assertEquals(infoMap.get(infoItem), infoColumns.get(infoItem));
+ }
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ ApplicationColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ ApplicationColumnPrefix.RELATES_TO.readResult(result,
+ relatesToKey).toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ ApplicationColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf.size(), configColumns.size());
+ for (String configItem : conf.keySet()) {
+ assertEquals(conf.get(configItem), configColumns.get(configItem));
+ }
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ // We got metrics back
+ assertNotNull(metricMap);
+ // Same number of metrics as we wrote
+ assertEquals(metricValues.entrySet().size(), metricMap.entrySet().size());
+
+ // Iterate over original metrics and confirm that they are present
+ // here.
+ for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
+ assertEquals(metricEntry.getValue(),
+ metricMap.get(metricEntry.getKey()));
+ }
+
+ TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, id,
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
+ Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
+ id, entity.getType(), null, null, null, null, null, null, null,
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ assertNotNull(e1);
+ assertEquals(1, es1.size());
+ } finally {
+ if (hbi != null) {
+ hbi.stop();
+ hbi.close();
+ }
+ if (hbr != null) {
+ hbr.stop();
+ hbr.close();
+ }
+ }
}
@Test
@@ -154,7 +360,7 @@ public class TestHBaseTimelineWriterImpl {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- String cluster = "cluster1";
+ String cluster = "cluster_test_write_entity";
String user = "user1";
String flow = "some_flow_name";
String flowVersion = "AB7822C10F1111";
@@ -268,7 +474,8 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(17, colCount);
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
@@ -284,10 +491,6 @@ public class TestHBaseTimelineWriterImpl {
hbr.close();
}
}
-
- // Somewhat of a hack, not a separate test in order not to have to deal with
- // test case order exectution.
- testAdditionalEntity();
}
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
@@ -299,14 +502,31 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.invert(runid), Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
return true;
}
- private void testAdditionalEntity() throws IOException {
+ private boolean isApplicationRowKeyCorrect(byte[] rowKey, String cluster,
+ String user, String flow, Long runid, String appName) {
+
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
+
+ assertTrue(rowKeyComponents.length == 5);
+ assertEquals(cluster, Bytes.toString(rowKeyComponents[0]));
+ assertEquals(user, Bytes.toString(rowKeyComponents[1]));
+ assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
+ assertEquals(TimelineWriterUtils.invert(runid),
+ Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
+ return true;
+ }
+
+ @Test
+ public void testEvents() throws IOException {
TimelineEvent event = new TimelineEvent();
String eventId = ApplicationMetricsConstants.CREATED_EVENT_TYPE;
event.setId(eventId);
@@ -333,7 +553,7 @@ public class TestHBaseTimelineWriterImpl {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- String cluster = "cluster2";
+ String cluster = "cluster_test_events";
String user = "user2";
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
@@ -341,50 +561,46 @@ public class TestHBaseTimelineWriterImpl {
String appName = "some app name";
hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
hbi.stop();
- // scan the table and see that entity exists
- Scan s = new Scan();
- byte[] startRow =
- EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
- s.setStartRow(startRow);
- Connection conn = ConnectionFactory.createConnection(c1);
- ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
-
- int rowCount = 0;
- for (Result result : scanner) {
- if (result != null && !result.isEmpty()) {
- rowCount++;
-
- // check the row key
- byte[] row1 = result.getRow();
- assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
- entity));
- Map<String, Object> eventsResult =
- EntityColumnPrefix.EVENT.readResults(result);
- // there should be only one event
- assertEquals(1, eventsResult.size());
- // key name for the event
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(Bytes.toBytes(eventId),
- Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
- Bytes.toBytes(expKey));
- String valueKey = Bytes.toString(compoundColumnQualifierBytes);
- for (Map.Entry<String, Object> e :
- eventsResult.entrySet()) {
- // the value key must match
- assertEquals(valueKey, e.getKey());
- Object value = e.getValue();
- // there should be only one timestamp and value
- assertEquals(expVal, value.toString());
- }
- }
+ // retrieve the row
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(cluster, user, flow, runid, appName);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Result result = new ApplicationTable().getResult(c1, conn, get);
+
+ assertTrue(result != null);
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isApplicationRowKeyCorrect(row1, cluster, user, flow, runid,
+ appName));
+
+ Map<String, Object> eventsResult =
+ ApplicationColumnPrefix.EVENT.readResults(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ // key name for the event
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(Bytes.toBytes(eventId),
+ Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
+ Bytes.toBytes(expKey));
+ String valueKey = Bytes.toString(compoundColumnQualifierBytes);
+ for (Map.Entry<String, Object> e : eventsResult.entrySet()) {
+ // the value key must match
+ assertEquals(valueKey, e.getKey());
+ Object value = e.getValue();
+ // there should be only one timestamp and value
+ assertEquals(expVal, value.toString());
}
- assertEquals(1, rowCount);
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
TimelineEntity e2 = hbr.getEntity(user, cluster, null, null, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
@@ -410,7 +626,7 @@ public class TestHBaseTimelineWriterImpl {
}
@Test
- public void testAdditionalEntityEmptyEventInfo() throws IOException {
+ public void testEventsWithEmptyInfo() throws IOException {
TimelineEvent event = new TimelineEvent();
String eventId = "foo_event_id";
event.setId(eventId);
@@ -435,7 +651,7 @@ public class TestHBaseTimelineWriterImpl {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- String cluster = "cluster_emptyeventkey";
+ String cluster = "cluster_test_empty_eventkey";
String user = "user_emptyeventkey";
String flow = "other_flow_name";
String flowVersion = "1111F01C2287BA";
@@ -487,7 +703,8 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(1, rowCount);
TimelineEntity e1 = hbr.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), EnumSet.of(TimelineReader.Field.ALL));
+ entity.getType(), entity.getId(),
+ EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = hbr.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
[39/50] [abbrv] hadoop git commit: YARN-3993. Changed to use the AM
flag in ContainerContext determine AM container in
TestPerNodeTimelineCollectorsAuxService. Contributed by Sunil G.
Posted by sj...@apache.org.
YARN-3993. Changed to use the AM flag in ContainerContext determine AM container in TestPerNodeTimelineCollectorsAuxService. Contributed by Sunil G.
(cherry picked from commit 9e48f9ff2ce08f3dcdd8d60bacb697664b92196f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d93e636d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d93e636d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d93e636d
Branch: refs/heads/YARN-2928
Commit: d93e636d1ddb4e63a47e0c43189abc6e6b57f672
Parents: 5b3b86a
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Aug 3 16:55:44 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../collector/PerNodeTimelineCollectorsAuxService.java | 13 +++----------
.../TestPerNodeTimelineCollectorsAuxService.java | 5 +++++
3 files changed, 11 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93e636d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 69112b5..f5ce32a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -118,6 +118,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3908. Fixed bugs in HBaseTimelineWriterImpl. (Vrushali C and Sangjin
Lee via zjshen)
+ YARN-3993. Changed to use the AM flag in ContainerContext determine AM
+ container in TestPerNodeTimelineCollectorsAuxService. (Sunil G via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93e636d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 3ede97a..befaa83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.server.api.AuxiliaryService;
import org.apache.hadoop.yarn.server.api.ContainerContext;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.api.ContainerType;
import com.google.common.annotations.VisibleForTesting;
@@ -119,7 +120,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
public void initializeContainer(ContainerInitializationContext context) {
// intercept the event of the AM container being created and initialize the
// app level collector service
- if (isApplicationMaster(context)) {
+ if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId();
addApplication(appId);
@@ -135,21 +136,13 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
public void stopContainer(ContainerTerminationContext context) {
// intercept the event of the AM container being stopped and remove the app
// level collector service
- if (isApplicationMaster(context)) {
+ if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
ApplicationId appId = context.getContainerId().
getApplicationAttemptId().getApplicationId();
removeApplication(appId);
}
}
- private boolean isApplicationMaster(ContainerContext context) {
- // TODO this is based on a (shaky) assumption that the container id (the
- // last field of the full container id) for an AM is always 1
- // we want to make this much more reliable
- ContainerId containerId = context.getContainerId();
- return containerId.getContainerId() == 1L;
- }
-
@VisibleForTesting
boolean hasApplication(ApplicationId appId) {
return collectorManager.containsTimelineCollector(appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d93e636d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index b1a5b04..7cc612d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
+import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.junit.After;
@@ -94,6 +95,8 @@ public class TestPerNodeTimelineCollectorsAuxService {
ContainerTerminationContext context =
mock(ContainerTerminationContext.class);
when(context.getContainerId()).thenReturn(containerId);
+ when(context.getContainerType()).thenReturn(
+ ContainerType.APPLICATION_MASTER);
auxService.stopContainer(context);
// auxService should not have that app
assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
@@ -138,6 +141,8 @@ public class TestPerNodeTimelineCollectorsAuxService {
ContainerInitializationContext context =
mock(ContainerInitializationContext.class);
when(context.getContainerId()).thenReturn(containerId);
+ when(context.getContainerType()).thenReturn(
+ ContainerType.APPLICATION_MASTER);
auxService.initializeContainer(context);
return auxService;
}
[21/50] [abbrv] hadoop git commit: YARN-3562. unit tests failures and
issues found from findbug from earlier ATS checkins (Naganarasimha G R via
sjlee)
Posted by sj...@apache.org.
YARN-3562. unit tests failures and issues found from findbug from earlier ATS checkins (Naganarasimha G R via sjlee)
(cherry picked from commit d4a23625b1e9a2c4cefd5fa68c28549ba6c1bc2e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed95a79f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed95a79f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed95a79f
Branch: refs/heads/YARN-2928
Commit: ed95a79fe5962c07a0aa9292963c061227e8c06b
Parents: 8414769
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed May 6 20:31:50 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:11 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/ResourceTrackerService.java | 4 ++--
.../timelineservice/RMTimelineCollectorManager.java | 2 +-
.../yarn/server/resourcemanager/TestAppManager.java | 5 +++++
.../server/resourcemanager/TestClientRMService.java | 12 +++++++++++-
.../TestRMAppLogAggregationStatus.java | 4 ++++
.../metrics/TestSystemMetricsPublisher.java | 2 +-
.../resourcemanager/rmapp/TestRMAppTransitions.java | 5 +++++
.../org/apache/hadoop/yarn/server/MiniYARNCluster.java | 2 ++
.../storage/FileSystemTimelineWriterImpl.java | 7 +++++--
10 files changed, 39 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 41ed1ec..82f95d6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -61,6 +61,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3551. Consolidate data model change according to the backend
implementation (Zhijie Shen via sjlee)
+ YARN-3562. unit tests failures and issues found from findbug from earlier
+ ATS checkins (Naganarasimha G R via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index ad38a71..b873752 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -538,8 +538,8 @@ public class ResourceTrackerService extends AbstractService implements
appId + " is not found in RMContext!");
} else {
String previousCollectorAddr = rmApp.getCollectorAddr();
- if (previousCollectorAddr == null ||
- previousCollectorAddr != collectorAddr) {
+ if (previousCollectorAddr == null
+ || !previousCollectorAddr.equals(collectorAddr)) {
// sending collector update event.
RMAppCollectorUpdateEvent event =
new RMAppCollectorUpdateEvent(appId, collectorAddr);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
index 25e0e0f..7d1b657 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -65,7 +65,7 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
break;
case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
collector.getTimelineEntityContext().setFlowRunId(
- Long.valueOf(parts[1]));
+ Long.parseLong(parts[1]));
break;
default:
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index cbeae5b..f163951 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAlloca
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -186,9 +187,11 @@ public class TestAppManager{
public int getCompletedAppsListSize() {
return super.getCompletedAppsListSize();
}
+
public int getCompletedAppsInStateStore() {
return this.completedAppsInStateStore;
}
+
public void submitApplication(
ApplicationSubmissionContext submissionContext, String user)
throws YarnException {
@@ -219,6 +222,8 @@ public class TestAppManager{
long now = System.currentTimeMillis();
rmContext = mockRMContext(1, now - 10);
+ rmContext
+ .setRMTimelineCollectorManager(mock(RMTimelineCollectorManager.class));
ResourceScheduler scheduler = mockResourceScheduler();
((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 8031759..d3fae6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -24,6 +24,7 @@ import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyBoolean;
import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -145,6 +146,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Clock;
@@ -842,6 +844,9 @@ public class TestClientRMService {
new EventHandler<Event>() {
public void handle(Event event) {}
});
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
ApplicationId appId1 = getApplicationId(100);
ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
@@ -929,6 +934,9 @@ public class TestClientRMService {
mockRMContext(yarnScheduler, rmContext);
RMStateStore stateStore = mock(RMStateStore.class);
when(rmContext.getStateStore()).thenReturn(stateStore);
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler,
null, mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
@@ -1100,7 +1108,9 @@ public class TestClientRMService {
};
when(rmContext.getDispatcher().getEventHandler()).thenReturn(eventHandler);
-
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
final ClientRMService rmService =
new ClientRMService(rmContext, yarnScheduler, appManager, null, null,
null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
index fccfa19..ae14253 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/logaggregationstatus/TestRMAppLogAggregationStatus.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -91,6 +92,9 @@ public class TestRMAppLogAggregationStatus {
rmContext.setSystemMetricsPublisher(new SystemMetricsPublisher());
rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
+ rmContext
+ .setRMTimelineCollectorManager(mock(RMTimelineCollectorManager.class));
+
scheduler = mock(YarnScheduler.class);
doAnswer(
new Answer<Void>() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index eb48cc7..e0d1b6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -72,7 +72,7 @@ public class TestSystemMetricsPublisher {
public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 2e64d61..a22a46e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
+
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doReturn;
@@ -80,6 +81,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSec
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
@@ -225,6 +227,9 @@ public class TestRMAppTransitions {
.getAppResourceUsageReport((ApplicationAttemptId)Matchers.any());
doReturn(resourceScheduler).when(rmContext).getScheduler();
+ doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext)
+ .getRMTimelineCollectorManager();
+
rmDispatcher.register(RMAppAttemptEventType.class,
new TestApplicationAttemptEventDispatcher(this.rmContext));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 319047d..991fcd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -522,6 +522,8 @@ public class MiniYARNCluster extends CompositeService {
MiniYARNCluster.getHostname() + ":0");
config.set(YarnConfiguration.NM_LOCALIZER_ADDRESS,
MiniYARNCluster.getHostname() + ":0");
+ config.set(YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ MiniYARNCluster.getHostname() + ":0");
WebAppUtils
.setNMWebAppHostNameAndPort(config,
MiniYARNCluster.getHostname(), 0);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed95a79f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index dd8ad06..ee1515d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -20,8 +20,9 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.BufferedWriter;
import java.io.File;
-import java.io.FileWriter;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -83,7 +84,9 @@ public class FileSystemTimelineWriterImpl extends AbstractService
String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowName,
flowVersion, String.valueOf(flowRun), appId, entity.getType());
String fileName = dir + entity.getId() + TIMELINE_SERVICE_STORAGE_EXTENSION;
- out = new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)));
+ out =
+ new PrintWriter(new BufferedWriter(new OutputStreamWriter(
+ new FileOutputStream(fileName, true), "UTF-8")));
out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
out.write("\n");
} catch (IOException ioe) {
[32/50] [abbrv] hadoop git commit: YARN-3706. Generalize native HBase
writer for additional tables (Joep Rottinghuis via sjlee)
Posted by sj...@apache.org.
YARN-3706. Generalize native HBase writer for additional tables (Joep Rottinghuis via sjlee)
(cherry picked from commit 9137aeae0dec83f9eff40d12cae712dfd508c0c5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42ed0625
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42ed0625
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42ed0625
Branch: refs/heads/YARN-2928
Commit: 42ed0625da34ef8d4794140289c102920259eaf0
Parents: 623b3ab
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Jun 18 10:49:20 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:13 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/EntityColumnDetails.java | 110 ------
.../storage/EntityColumnFamily.java | 95 -----
.../storage/HBaseTimelineWriterImpl.java | 114 +++---
.../server/timelineservice/storage/Range.java | 59 ----
.../storage/TimelineEntitySchemaConstants.java | 71 ----
.../storage/TimelineSchemaCreator.java | 134 +-------
.../timelineservice/storage/TimelineWriter.java | 3 +-
.../storage/TimelineWriterUtils.java | 344 -------------------
.../storage/common/BaseTable.java | 118 +++++++
.../common/BufferedMutatorDelegator.java | 73 ++++
.../timelineservice/storage/common/Column.java | 59 ++++
.../storage/common/ColumnFamily.java | 34 ++
.../storage/common/ColumnHelper.java | 247 +++++++++++++
.../storage/common/ColumnPrefix.java | 83 +++++
.../timelineservice/storage/common/Range.java | 59 ++++
.../storage/common/Separator.java | 303 ++++++++++++++++
.../common/TimelineEntitySchemaConstants.java | 68 ++++
.../storage/common/TimelineWriterUtils.java | 127 +++++++
.../storage/common/TypedBufferedMutator.java | 28 ++
.../storage/common/package-info.java | 24 ++
.../storage/entity/EntityColumn.java | 141 ++++++++
.../storage/entity/EntityColumnFamily.java | 65 ++++
.../storage/entity/EntityColumnPrefix.java | 212 ++++++++++++
.../storage/entity/EntityRowKey.java | 93 +++++
.../storage/entity/EntityTable.java | 161 +++++++++
.../storage/entity/package-info.java | 25 ++
.../storage/TestHBaseTimelineWriterImpl.java | 252 ++++++++------
.../storage/common/TestSeparator.java | 129 +++++++
.../storage/common/TestTimelineWriterUtils.java | 29 ++
30 files changed, 2301 insertions(+), 962 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1cf1f26..1e85cdf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -93,6 +93,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
zjshen)
+ YARN-3706. Generalize native HBase writer for additional tables (Joep
+ Rottinghuis via sjlee)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
deleted file mode 100644
index 2894c41..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnDetails.java
+++ /dev/null
@@ -1,110 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Contains the Info Column Family details like Column names, types and byte
- * representations for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * object that is stored in hbase Also has utility functions for storing each of
- * these to the backend
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-enum EntityColumnDetails {
- ID(EntityColumnFamily.INFO, "id"),
- TYPE(EntityColumnFamily.INFO, "type"),
- CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
- MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
- FLOW_VERSION(EntityColumnFamily.INFO, "flow_version"),
- PREFIX_IS_RELATED_TO(EntityColumnFamily.INFO, "r"),
- PREFIX_RELATES_TO(EntityColumnFamily.INFO, "s"),
- PREFIX_EVENTS(EntityColumnFamily.INFO, "e");
-
- private final EntityColumnFamily columnFamily;
- private final String value;
- private final byte[] inBytes;
-
- private EntityColumnDetails(EntityColumnFamily columnFamily,
- String value) {
- this.columnFamily = columnFamily;
- this.value = value;
- this.inBytes = Bytes.toBytes(this.value.toLowerCase());
- }
-
- public String getValue() {
- return value;
- }
-
- byte[] getInBytes() {
- return inBytes;
- }
-
- void store(byte[] rowKey, BufferedMutator entityTable, Object inputValue)
- throws IOException {
- TimelineWriterUtils.store(rowKey, entityTable,
- this.columnFamily.getInBytes(), null, this.getInBytes(), inputValue,
- null);
- }
-
- /**
- * stores events data with column prefix
- */
- void store(byte[] rowKey, BufferedMutator entityTable, byte[] idBytes,
- String key, Object inputValue) throws IOException {
- TimelineWriterUtils.store(rowKey, entityTable,
- this.columnFamily.getInBytes(),
- // column prefix
- TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
- this.getInBytes(), idBytes),
- // column qualifier
- Bytes.toBytes(key),
- inputValue, null);
- }
-
- /**
- * stores relation entities with a column prefix
- */
- void store(byte[] rowKey, BufferedMutator entityTable, String key,
- Set<String> inputValue) throws IOException {
- TimelineWriterUtils.store(rowKey, entityTable,
- this.columnFamily.getInBytes(),
- // column prefix
- this.getInBytes(),
- // column qualifier
- Bytes.toBytes(key),
- // value
- TimelineWriterUtils.getValueAsString(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, inputValue),
- // cell timestamp
- null);
- }
-
- // TODO add a method that accepts a byte array,
- // iterates over the enum and returns an enum from those bytes
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
deleted file mode 100644
index e556351..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/EntityColumnFamily.java
+++ /dev/null
@@ -1,95 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.util.Bytes;
-
-/**
- * Contains the Column family names and byte representations for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * object that is stored in hbase
- * Also has utility functions for storing each of these to the backend
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-enum EntityColumnFamily {
- INFO("i"),
- CONFIG("c"),
- METRICS("m");
-
- private final String value;
- private final byte[] inBytes;
-
- private EntityColumnFamily(String value) {
- this.value = value;
- this.inBytes = Bytes.toBytes(this.value.toLowerCase());
- }
-
- byte[] getInBytes() {
- return inBytes;
- }
-
- public String getValue() {
- return value;
- }
-
- /**
- * stores the key as column and value as hbase column value in the given
- * column family in the entity table
- *
- * @param rowKey
- * @param entityTable
- * @param inputValue
- * @throws IOException
- */
- public void store(byte[] rowKey, BufferedMutator entityTable, String key,
- String inputValue) throws IOException {
- if (key == null) {
- return;
- }
- TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
- Bytes.toBytes(key), inputValue, null);
- }
-
- /**
- * stores the values along with cell timestamp
- *
- * @param rowKey
- * @param entityTable
- * @param key
- * @param timestamp
- * @param inputValue
- * @throws IOException
- */
- public void store(byte[] rowKey, BufferedMutator entityTable, String key,
- Long timestamp, Number inputValue) throws IOException {
- if (key == null) {
- return;
- }
- TimelineWriterUtils.store(rowKey, entityTable, this.getInBytes(), null,
- Bytes.toBytes(key), inputValue, timestamp);
- }
-
- // TODO add a method that accepts a byte array,
- // iterates over the enum and returns an enum from those bytes
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index aa71c6c..e48ca60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -26,19 +26,22 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineEntitySchemaConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
/**
* This implements a hbase based backend for storing application timeline entity
@@ -50,7 +53,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
TimelineWriter {
private Connection conn;
- private BufferedMutator entityTable;
+ private TypedBufferedMutator<EntityTable> entityTable;
private static final Log LOG = LogFactory
.getLog(HBaseTimelineWriterImpl.class);
@@ -72,10 +75,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
super.serviceInit(conf);
Configuration hbaseConf = HBaseConfiguration.create(conf);
conn = ConnectionFactory.createConnection(hbaseConf);
- TableName entityTableName = TableName.valueOf(hbaseConf.get(
- TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
- TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
- entityTable = conn.getBufferedMutator(entityTableName);
+ entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -86,9 +86,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
String flowName, String flowVersion, long flowRunId, String appId,
TimelineEntities data) throws IOException {
- byte[] rowKeyPrefix = TimelineWriterUtils.getRowKeyPrefix(clusterId,
- userId, flowName, flowRunId, appId);
-
TimelineWriteResponse putStatus = new TimelineWriteResponse();
for (TimelineEntity te : data.getEntities()) {
@@ -96,19 +93,19 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (te == null) {
continue;
}
- // get row key
- byte[] row = TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, rowKeyPrefix,
- Bytes.toBytes(te.getType()), Bytes.toBytes(te.getId()));
-
- storeInfo(row, te, flowVersion);
- storeEvents(row, te.getEvents());
- storeConfig(row, te.getConfigs());
- storeMetrics(row, te.getMetrics());
- storeRelations(row, te.getIsRelatedToEntities(),
- EntityColumnDetails.PREFIX_IS_RELATED_TO);
- storeRelations(row, te.getRelatesToEntities(),
- EntityColumnDetails.PREFIX_RELATES_TO);
+
+ byte[] rowKey =
+ EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
+ te);
+
+ storeInfo(rowKey, te, flowVersion);
+ storeEvents(rowKey, te.getEvents());
+ storeConfig(rowKey, te.getConfigs());
+ storeMetrics(rowKey, te.getMetrics());
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ EntityColumnPrefix.IS_RELATED_TO);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ EntityColumnPrefix.RELATES_TO);
}
return putStatus;
@@ -119,10 +116,15 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
*/
private void storeRelations(byte[] rowKey,
Map<String, Set<String>> connectedEntities,
- EntityColumnDetails columnNamePrefix) throws IOException {
- for (Map.Entry<String, Set<String>> entry : connectedEntities.entrySet()) {
- columnNamePrefix.store(rowKey, entityTable, entry.getKey(),
- entry.getValue());
+ EntityColumnPrefix entityColumnPrefix) throws IOException {
+ for (Map.Entry<String, Set<String>> connectedEntity : connectedEntities
+ .entrySet()) {
+ // id3?id4?id5
+ String compoundValue =
+ Separator.VALUES.joinEncoded(connectedEntity.getValue());
+
+ entityColumnPrefix.store(rowKey, entityTable, connectedEntity.getKey(),
+ null, compoundValue);
}
}
@@ -132,13 +134,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion)
throws IOException {
- EntityColumnDetails.ID.store(rowKey, entityTable, te.getId());
- EntityColumnDetails.TYPE.store(rowKey, entityTable, te.getType());
- EntityColumnDetails.CREATED_TIME.store(rowKey, entityTable,
+ EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
+ EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
+ EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
te.getCreatedTime());
- EntityColumnDetails.MODIFIED_TIME.store(rowKey, entityTable,
+ EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
te.getModifiedTime());
- EntityColumnDetails.FLOW_VERSION.store(rowKey, entityTable, flowVersion);
+ EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
}
/**
@@ -150,8 +152,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
return;
}
for (Map.Entry<String, String> entry : config.entrySet()) {
- EntityColumnFamily.CONFIG.store(rowKey, entityTable,
- entry.getKey(), entry.getValue());
+ EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
+ null, entry.getValue());
}
}
@@ -163,11 +165,12 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
throws IOException {
if (metrics != null) {
for (TimelineMetric metric : metrics) {
- String key = metric.getId();
+ String metricColumnQualifier = metric.getId();
Map<Long, Number> timeseries = metric.getValues();
- for (Map.Entry<Long, Number> entry : timeseries.entrySet()) {
- EntityColumnFamily.METRICS.store(rowKey, entityTable, key,
- entry.getKey(), entry.getValue());
+ for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
+ Long timestamp = timeseriesEntry.getKey();
+ EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
}
}
}
@@ -181,19 +184,27 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
if (events != null) {
for (TimelineEvent event : events) {
if (event != null) {
- String id = event.getId();
- if (id != null) {
- byte[] idBytes = Bytes.toBytes(id);
+ String eventId = event.getId();
+ if (eventId != null) {
Map<String, Object> eventInfo = event.getInfo();
if (eventInfo != null) {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
- EntityColumnDetails.PREFIX_EVENTS.store(rowKey,
- entityTable, idBytes, info.getKey(), info.getValue());
- }
+ // eventId?infoKey
+ byte[] columnQualifierFirst =
+ Bytes.toBytes(Separator.VALUES.encode(eventId));
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(columnQualifierFirst,
+ Bytes.toBytes(info.getKey()));
+ // convert back to string to avoid additional API on store.
+ String compoundColumnQualifier =
+ Bytes.toString(compoundColumnQualifierBytes);
+ EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ compoundColumnQualifier, null, info.getValue());
+ } // for info: eventInfo
}
}
}
- }
+ } // event : events
}
}
@@ -204,8 +215,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * close the hbase connections
- * The close APIs perform flushing and release any
+ * close the hbase connections The close APIs perform flushing and release any
* resources held
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
deleted file mode 100644
index 2a2db81..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/Range.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class Range {
- private final int startIdx;
- private final int endIdx;
-
- /**
- * Defines a range from start index (inclusive) to end index (exclusive).
- *
- * @param start
- * Starting index position
- * @param end
- * Ending index position (exclusive)
- */
- public Range(int start, int end) {
- if (start < 0 || end < start) {
- throw new IllegalArgumentException(
- "Invalid range, required that: 0 <= start <= end; start=" + start
- + ", end=" + end);
- }
-
- this.startIdx = start;
- this.endIdx = end;
- }
-
- public int start() {
- return startIdx;
- }
-
- public int end() {
- return endIdx;
- }
-
- public int length() {
- return endIdx - startIdx;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
deleted file mode 100644
index d95cbb2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntitySchemaConstants.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-
-/**
- * contains the constants used in the context of schema accesses for
- * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * information
- */
-@InterfaceAudience.Private
-@InterfaceStability.Unstable
-public class TimelineEntitySchemaConstants {
-
- /** entity prefix */
- public static final String ENTITY_PREFIX =
- YarnConfiguration.TIMELINE_SERVICE_PREFIX
- + ".entity";
-
- /** config param name that specifies the entity table name */
- public static final String ENTITY_TABLE_NAME = ENTITY_PREFIX
- + ".table.name";
-
- /**
- * config param name that specifies the TTL for metrics column family in
- * entity table
- */
- public static final String ENTITY_TABLE_METRICS_TTL = ENTITY_PREFIX
- + ".table.metrics.ttl";
-
- /** default value for entity table name */
- public static final String DEFAULT_ENTITY_TABLE_NAME = "timelineservice.entity";
-
- /** in bytes default value for entity table name */
- static final byte[] DEFAULT_ENTITY_TABLE_NAME_BYTES = Bytes
- .toBytes(DEFAULT_ENTITY_TABLE_NAME);
-
- /** separator in row key */
- public static final String ROW_KEY_SEPARATOR = "!";
-
- /** byte representation of the separator in row key */
- static final byte[] ROW_KEY_SEPARATOR_BYTES = Bytes
- .toBytes(ROW_KEY_SEPARATOR);
-
- public static final byte ZERO_BYTES = 0;
-
- /** default TTL is 30 days for metrics timeseries */
- public static final int ENTITY_TABLE_METRICS_TTL_DEFAULT = 2592000;
-
- /** default max number of versions */
- public static final int ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT = 1000;
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 820a6d1..a5cc2ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -19,21 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.regionserver.BloomType;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.HelpFormatter;
@@ -41,7 +26,18 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.cli.PosixParser;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
/**
* This creates the schema for a hbase based backend for storing application
@@ -53,18 +49,6 @@ public class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
- final static byte[][] splits = { Bytes.toBytes("a"), Bytes.toBytes("ad"),
- Bytes.toBytes("an"), Bytes.toBytes("b"), Bytes.toBytes("ca"),
- Bytes.toBytes("cl"), Bytes.toBytes("d"), Bytes.toBytes("e"),
- Bytes.toBytes("f"), Bytes.toBytes("g"), Bytes.toBytes("h"),
- Bytes.toBytes("i"), Bytes.toBytes("j"), Bytes.toBytes("k"),
- Bytes.toBytes("l"), Bytes.toBytes("m"), Bytes.toBytes("n"),
- Bytes.toBytes("o"), Bytes.toBytes("q"), Bytes.toBytes("r"),
- Bytes.toBytes("s"), Bytes.toBytes("se"), Bytes.toBytes("t"),
- Bytes.toBytes("u"), Bytes.toBytes("v"), Bytes.toBytes("w"),
- Bytes.toBytes("x"), Bytes.toBytes("y"), Bytes.toBytes("z") };
-
- public static final String SPLIT_KEY_PREFIX_LENGTH = "4";
public static void main(String[] args) throws Exception {
@@ -79,13 +63,12 @@ public class TimelineSchemaCreator {
// Grab the entityTableName argument
String entityTableName = commandLine.getOptionValue("e");
if (StringUtils.isNotBlank(entityTableName)) {
- hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
- entityTableName);
+ hbaseConf.set(EntityTable.TABLE_NAME_CONF_NAME, entityTableName);
}
- String entityTable_TTL_Metrics = commandLine.getOptionValue("m");
- if (StringUtils.isNotBlank(entityTable_TTL_Metrics)) {
- hbaseConf.set(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
- entityTable_TTL_Metrics);
+ String entityTableTTLMetrics = commandLine.getOptionValue("m");
+ if (StringUtils.isNotBlank(entityTableTTLMetrics)) {
+ int metricsTTL = Integer.parseInt(entityTableTTLMetrics);
+ new EntityTable().setMetricsTTL(metricsTTL, hbaseConf);
}
createAllTables(hbaseConf);
}
@@ -136,7 +119,7 @@ public class TimelineSchemaCreator {
if (admin == null) {
throw new IOException("Cannot create table since admin is null");
}
- createTimelineEntityTable(admin, hbaseConf);
+ new EntityTable().createTable(admin, hbaseConf);
} finally {
if (conn != null) {
conn.close();
@@ -144,88 +127,5 @@ public class TimelineSchemaCreator {
}
}
- /**
- * Creates a table with column families info, config and metrics
- * info stores information about a timeline entity object
- * config stores configuration data of a timeline entity object
- * metrics stores the metrics of a timeline entity object
- *
- * Example entity table record:
- * <pre>
- *|------------------------------------------------------------|
- *| Row | Column Family | Column Family | Column Family|
- *| key | info | metrics | config |
- *|------------------------------------------------------------|
- *| userName! | id:entityId | metricName1: | configKey1: |
- *| clusterId! | | metricValue1 | configValue1 |
- *| flowId! | type:entityType| @timestamp1 | |
- *| flowRunId! | | | configKey2: |
- *| AppId! | created_time: | metricName1: | configValue2 |
- *| entityType!| 1392993084018 | metricValue2 | |
- *| entityId | | @timestamp2 | |
- *| | modified_time: | | |
- *| | 1392995081012 | metricName2: | |
- *| | | metricValue1 | |
- *| | r!relatesToKey:| @timestamp2 | |
- *| | id3!id4!id5 | | |
- *| | | | |
- *| | s!isRelatedToKey| | |
- *| | id7!id9!id5 | | |
- *| | | | |
- *| | e!eventKey: | | |
- *| | eventValue | | |
- *| | | | |
- *| | flowVersion: | | |
- *| | versionValue | | |
- *|------------------------------------------------------------|
- *</pre>
- * @param admin
- * @param hbaseConf
- * @throws IOException
- */
- public static void createTimelineEntityTable(Admin admin,
- Configuration hbaseConf) throws IOException {
-
- TableName table = TableName.valueOf(hbaseConf.get(
- TimelineEntitySchemaConstants.ENTITY_TABLE_NAME,
- TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME));
- if (admin.tableExists(table)) {
- // do not disable / delete existing table
- // similar to the approach taken by map-reduce jobs when
- // output directory exists
- throw new IOException("Table " + table.getNameAsString()
- + " already exists.");
- }
-
- HTableDescriptor entityTableDescp = new HTableDescriptor(table);
- HColumnDescriptor cf1 = new HColumnDescriptor(
- EntityColumnFamily.INFO.getInBytes());
- cf1.setBloomFilterType(BloomType.ROWCOL);
- entityTableDescp.addFamily(cf1);
-
- HColumnDescriptor cf2 = new HColumnDescriptor(
- EntityColumnFamily.CONFIG.getInBytes());
- cf2.setBloomFilterType(BloomType.ROWCOL);
- cf2.setBlockCacheEnabled(true);
- entityTableDescp.addFamily(cf2);
-
- HColumnDescriptor cf3 = new HColumnDescriptor(
- EntityColumnFamily.METRICS.getInBytes());
- entityTableDescp.addFamily(cf3);
- cf3.setBlockCacheEnabled(true);
- // always keep 1 version (the latest)
- cf3.setMinVersions(1);
- cf3.setMaxVersions(TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_MAX_VERSIONS_DEFAULT);
- cf3.setTimeToLive(hbaseConf.getInt(
- TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL,
- TimelineEntitySchemaConstants.ENTITY_TABLE_METRICS_TTL_DEFAULT));
- entityTableDescp
- .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
- entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
- SPLIT_KEY_PREFIX_LENGTH);
- admin.createTable(entityTableDescp, splits);
- LOG.info("Status of table creation for " + table.getNameAsString() + "="
- + admin.tableExists(table));
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 467bcec..494e8ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -15,17 +15,16 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.Service;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.service.Service;
/**
* This interface is for storing application timeline information.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
deleted file mode 100644
index 113935e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriterUtils.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
-import org.apache.hadoop.hbase.client.BufferedMutator;
-import org.apache.hadoop.hbase.client.Put;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.timelineservice.storage.Range;
-
-/**
- * bunch of utility functions used across TimelineWriter classes
- */
-@InterfaceAudience.Public
-@InterfaceStability.Unstable
-public class TimelineWriterUtils {
-
- /** empty bytes */
- public static final byte[] EMPTY_BYTES = new byte[0];
- private static final String SPACE = " ";
- private static final String UNDERSCORE = "_";
- private static final String EMPTY_STRING = "";
-
- /**
- * Returns a single byte array containing all of the individual component
- * arrays separated by the separator array.
- *
- * @param separator
- * @param components
- * @return byte array after joining the components
- */
- public static byte[] join(byte[] separator, byte[]... components) {
- if (components == null || components.length == 0) {
- return EMPTY_BYTES;
- }
-
- int finalSize = 0;
- if (separator != null) {
- finalSize = separator.length * (components.length - 1);
- }
- for (byte[] comp : components) {
- if (comp != null) {
- finalSize += comp.length;
- }
- }
-
- byte[] buf = new byte[finalSize];
- int offset = 0;
- for (int i = 0; i < components.length; i++) {
- if (components[i] != null) {
- System.arraycopy(components[i], 0, buf, offset, components[i].length);
- offset += components[i].length;
- if (i < (components.length - 1) && separator != null
- && separator.length > 0) {
- System.arraycopy(separator, 0, buf, offset, separator.length);
- offset += separator.length;
- }
- }
- }
- return buf;
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments. To identify the split
- * ranges without the array copies, see
- * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
- *
- * @param source
- * @param separator
- * @return byte[] array after splitting the source
- */
- public static byte[][] split(byte[] source, byte[] separator) {
- return split(source, separator, -1);
- }
-
- /**
- * Splits the source array into multiple array segments using the given
- * separator, up to a maximum of count items. This will naturally produce
- * copied byte arrays for each of the split segments. To identify the split
- * ranges without the array copies, see
- * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
- *
- * @param source
- * @param separator
- * @param limit
- * @return byte[][] after splitting the input source
- */
- public static byte[][] split(byte[] source, byte[] separator, int limit) {
- List<Range> segments = splitRanges(source, separator, limit);
-
- byte[][] splits = new byte[segments.size()][];
- for (int i = 0; i < segments.size(); i++) {
- Range r = segments.get(i);
- byte[] tmp = new byte[r.length()];
- if (tmp.length > 0) {
- System.arraycopy(source, r.start(), tmp, 0, r.length());
- }
- splits[i] = tmp;
- }
- return splits;
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- */
- public static List<Range> splitRanges(byte[] source, byte[] separator) {
- return splitRanges(source, separator, -1);
- }
-
- /**
- * Returns a list of ranges identifying [start, end) -- closed, open --
- * positions within the source byte array that would be split using the
- * separator byte array.
- * @param source the source data
- * @param separator the separator pattern to look for
- * @param limit the maximum number of splits to identify in the source
- */
- public static List<Range> splitRanges(byte[] source, byte[] separator, int limit) {
- List<Range> segments = new ArrayList<Range>();
- if ((source == null) || (separator == null)) {
- return segments;
- }
- int start = 0;
- itersource: for (int i = 0; i < source.length; i++) {
- for (int j = 0; j < separator.length; j++) {
- if (source[i + j] != separator[j]) {
- continue itersource;
- }
- }
- // all separator elements matched
- if (limit > 0 && segments.size() >= (limit-1)) {
- // everything else goes in one final segment
- break;
- }
-
- segments.add(new Range(start, i));
- start = i + separator.length;
- // i will be incremented again in outer for loop
- i += separator.length-1;
- }
- // add in remaining to a final range
- if (start <= source.length) {
- segments.add(new Range(start, source.length));
- }
- return segments;
- }
-
- /**
- * converts run id into it's inverse timestamp
- * @param flowRunId
- * @return inverted long
- */
- public static long encodeRunId(Long flowRunId) {
- return Long.MAX_VALUE - flowRunId;
- }
-
- /**
- * return a value from the Map as a String
- * @param key
- * @param values
- * @return value as a String or ""
- * @throws IOException
- */
- public static String getValueAsString(final byte[] key,
- final Map<byte[], byte[]> values) throws IOException {
- if( values == null ) {
- return EMPTY_STRING;
- }
- byte[] value = values.get(key);
- if (value != null) {
- return GenericObjectMapper.read(value).toString();
- } else {
- return EMPTY_STRING;
- }
- }
-
- /**
- * return a value from the Map as a long
- * @param key
- * @param values
- * @return value as Long or 0L
- * @throws IOException
- */
- public static long getValueAsLong(final byte[] key,
- final Map<byte[], byte[]> values) throws IOException {
- if (values == null) {
- return 0;
- }
- byte[] value = values.get(key);
- if (value != null) {
- Number val = (Number) GenericObjectMapper.read(value);
- return val.longValue();
- } else {
- return 0L;
- }
- }
-
- /**
- * concates the values from a Set<Strings> to return a single delimited string value
- * @param rowKeySeparator
- * @param values
- * @return Value from the set of strings as a string
- */
- public static String getValueAsString(String rowKeySeparator,
- Set<String> values) {
-
- if (values == null) {
- return EMPTY_STRING;
- }
- StringBuilder concatStrings = new StringBuilder();
- for (String value : values) {
- concatStrings.append(value);
- concatStrings.append(rowKeySeparator);
- }
- // remove the last separator
- if(concatStrings.length() > 1) {
- concatStrings.deleteCharAt(concatStrings.lastIndexOf(rowKeySeparator));
- }
- return concatStrings.toString();
- }
- /**
- * Constructs a row key prefix for the entity table
- * @param clusterId
- * @param userId
- * @param flowId
- * @param flowRunId
- * @param appId
- * @return byte array with the row key prefix
- */
- static byte[] getRowKeyPrefix(String clusterId, String userId, String flowId,
- Long flowRunId, String appId) {
- return TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
- Bytes.toBytes(cleanse(userId)), Bytes.toBytes(cleanse(clusterId)),
- Bytes.toBytes(cleanse(flowId)),
- Bytes.toBytes(TimelineWriterUtils.encodeRunId(flowRunId)),
- Bytes.toBytes(cleanse(appId)));
- }
-
- /**
- * Takes a string token to be used as a key or qualifier and
- * cleanses out reserved tokens.
- * This operation is not symmetrical.
- * Logic is to replace all spaces and separator chars in input with
- * underscores.
- *
- * @param token token to cleanse.
- * @return String with no spaces and no separator chars
- */
- public static String cleanse(String token) {
- if (token == null || token.length() == 0) {
- return token;
- }
-
- String cleansed = token.replaceAll(SPACE, UNDERSCORE);
- cleansed = cleansed.replaceAll(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR, UNDERSCORE);
-
- return cleansed;
- }
-
- /**
- * stores the info to the table in hbase
- *
- * @param rowKey
- * @param table
- * @param columnFamily
- * @param columnPrefix
- * @param columnQualifier
- * @param inputValue
- * @param cellTimeStamp
- * @throws IOException
- */
- public static void store(byte[] rowKey, BufferedMutator table, byte[] columnFamily,
- byte[] columnPrefix, byte[] columnQualifier, Object inputValue,
- Long cellTimeStamp) throws IOException {
- if ((rowKey == null) || (table == null) || (columnFamily == null)
- || (columnQualifier == null) || (inputValue == null)) {
- return;
- }
-
- Put p = null;
- if (cellTimeStamp == null) {
- if (columnPrefix != null) {
- // store with prefix
- p = new Put(rowKey);
- p.addColumn(
- columnFamily,
- join(TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES,
- columnPrefix, columnQualifier), GenericObjectMapper
- .write(inputValue));
- } else {
- // store without prefix
- p = new Put(rowKey);
- p.addColumn(columnFamily, columnQualifier,
- GenericObjectMapper.write(inputValue));
- }
- } else {
- // store with cell timestamp
- Cell cell = CellUtil.createCell(rowKey, columnFamily, columnQualifier,
- // set the cell timestamp
- cellTimeStamp,
- // KeyValue Type minimum
- TimelineEntitySchemaConstants.ZERO_BYTES,
- GenericObjectMapper.write(inputValue));
- p = new Put(rowKey);
- p.add(cell);
- }
- if (p != null) {
- table.mutate(p);
- }
-
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
new file mode 100644
index 0000000..e8d8b5c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+
+/**
+ * Implements behavior common to tables used in the timeline service storage.
+ *
+ * @param <T> reference to the table instance class itself for type safety.
+ */
+public abstract class BaseTable<T> {
+
+ /**
+ * Name of config variable that is used to point to this table
+ */
+ private final String tableNameConfName;
+
+ /**
+ * Unless the configuration overrides, this will be the default name for the
+ * table when it is created.
+ */
+ private final String defaultTableName;
+
+ /**
+ * @param tableNameConfName name of config variable that is used to point to
+ * this table.
+ */
+ protected BaseTable(String tableNameConfName, String defaultTableName) {
+ this.tableNameConfName = tableNameConfName;
+ this.defaultTableName = defaultTableName;
+ }
+
+ /**
+ * Used to create a type-safe mutator for this table.
+ *
+ * @param hbaseConf used to read table name
+ * @param conn used to create a table from.
+ * @return a type safe {@link BufferedMutator} for the entity table.
+ * @throws IOException
+ */
+ public TypedBufferedMutator<T> getTableMutator(Configuration hbaseConf,
+ Connection conn) throws IOException {
+
+ TableName tableName = this.getTableName(hbaseConf);
+
+ // Plain buffered mutator
+ BufferedMutator bufferedMutator = conn.getBufferedMutator(tableName);
+
+ // Now make this thing type safe.
+ // This is how service initialization should hang on to this variable, with
+ // the proper type
+ TypedBufferedMutator<T> table =
+ new BufferedMutatorDelegator<T>(bufferedMutator);
+
+ return table;
+ }
+
+ /**
+ * @param hbaseConf used to read settings that override defaults
+ * @param conn used to create table from
+ * @param scan that specifies what you want to read from this table.
+ * @return scanner for the table.
+ * @throws IOException
+ */
+ public ResultScanner getResultScanner(Configuration hbaseConf,
+ Connection conn, Scan scan) throws IOException {
+ Table table = conn.getTable(getTableName(hbaseConf));
+ return table.getScanner(scan);
+ }
+
+ /**
+ * Get the table name for this table.
+ *
+ * @param hbaseConf
+ */
+ public TableName getTableName(Configuration hbaseConf) {
+ TableName table =
+ TableName.valueOf(hbaseConf.get(tableNameConfName, defaultTableName));
+ return table;
+
+ }
+
+ /**
+ * Used to create the table in HBase. Should be called only once (per HBase
+ * instance).
+ *
+ * @param admin
+ * @param hbaseConf
+ */
+ public abstract void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException;
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
new file mode 100644
index 0000000..fe8f9c6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
@@ -0,0 +1,73 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.BufferedMutator;
+import org.apache.hadoop.hbase.client.Mutation;
+
+/**
+ * To be used to wrap an actual {@link BufferedMutator} in a type safe manner
+ *
+ * @param <T> The class referring to the table to be written to.
+ */
+class BufferedMutatorDelegator<T> implements TypedBufferedMutator<T> {
+
+ private final BufferedMutator bufferedMutator;
+
+ /**
+ * @param bufferedMutator the mutator to be wrapped for delegation. Shall not
+ * be null.
+ */
+ public BufferedMutatorDelegator(BufferedMutator bufferedMutator) {
+ this.bufferedMutator = bufferedMutator;
+ }
+
+ public TableName getName() {
+ return bufferedMutator.getName();
+ }
+
+ public Configuration getConfiguration() {
+ return bufferedMutator.getConfiguration();
+ }
+
+ public void mutate(Mutation mutation) throws IOException {
+ bufferedMutator.mutate(mutation);
+ }
+
+ public void mutate(List<? extends Mutation> mutations) throws IOException {
+ bufferedMutator.mutate(mutations);
+ }
+
+ public void close() throws IOException {
+ bufferedMutator.close();
+ }
+
+ public void flush() throws IOException {
+ bufferedMutator.flush();
+ }
+
+ public long getWriteBufferSize() {
+ return bufferedMutator.getWriteBufferSize();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
new file mode 100644
index 0000000..3397d62
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Result;
+
+/**
+ * A Column represents the way to store a fully qualified column in a specific
+ * table.
+ */
+public interface Column<T> {
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
+ Long timestamp, Object inputValue) throws IOException;
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link Cell}.
+ *
+ * @param result Cannot be null
+ * @return result object (can be cast to whatever object was written to), or
+ * null when result doesn't contain this column.
+ * @throws IOException
+ */
+ public Object readResult(Result result) throws IOException;
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
new file mode 100644
index 0000000..c84c016
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+/**
+ * Type safe column family.
+ *
+ * @param <T> refers to the table for which this column family is used for.
+ */
+public interface ColumnFamily<T> {
+
+ /**
+ * Keep a local copy if you need to avoid overhead of repeated cloning.
+ *
+ * @return a clone of the byte representation of the column family.
+ */
+ public byte[] getBytes();
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
new file mode 100644
index 0000000..6a204dc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -0,0 +1,247 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+
+/**
+ * This class is meant to be used only by explicit Columns, and not directly to
+ * write by clients.
+ *
+ * @param <T> refers to the table.
+ */
+public class ColumnHelper<T> {
+
+ private final ColumnFamily<T> columnFamily;
+
+ /**
+ * Local copy of bytes representation of columnFamily so that we can avoid
+ * cloning a new copy over and over.
+ */
+ private final byte[] columnFamilyBytes;
+
+ public ColumnHelper(ColumnFamily<T> columnFamily) {
+ this.columnFamily = columnFamily;
+ columnFamilyBytes = columnFamily.getBytes();
+ }
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table
+ * @param columnQualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
+ byte[] columnQualifier, Long timestamp, Object inputValue)
+ throws IOException {
+ if ((rowKey == null) || (columnQualifier == null) || (inputValue == null)) {
+ return;
+ }
+ Put p = new Put(rowKey);
+
+ if (timestamp == null) {
+ p.addColumn(columnFamilyBytes, columnQualifier,
+ GenericObjectMapper.write(inputValue));
+ } else {
+ p.addColumn(columnFamilyBytes, columnQualifier, timestamp,
+ GenericObjectMapper.write(inputValue));
+ }
+ tableMutator.mutate(p);
+ }
+
+ /**
+ * @return the column family for this column implementation.
+ */
+ public ColumnFamily<T> getColumnFamily() {
+ return columnFamily;
+ }
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link Cell}.
+ *
+ * @param result from which to read the value. Cannot be null
+ * @param columnQualifierBytes referring to the column to be read.
+ * @return latest version of the specified column of whichever object was
+ * written.
+ * @throws IOException
+ */
+ public Object readResult(Result result, byte[] columnQualifierBytes)
+ throws IOException {
+ if (result == null || columnQualifierBytes == null) {
+ return null;
+ }
+
+ // Would have preferred to be able to use getValueAsByteBuffer and get a
+ // ByteBuffer to avoid copy, but GenericObjectMapper doesn't seem to like
+ // that.
+ byte[] value = result.getValue(columnFamilyBytes, columnQualifierBytes);
+ return GenericObjectMapper.read(value);
+ }
+
+ /**
+ * @param result from which to reads timeseries data
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @return the cell values at each respective time in for form
+ * {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}
+ * @throws IOException
+ */
+ public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
+ Result result, byte[] columnPrefixBytes) throws IOException {
+
+ NavigableMap<String, NavigableMap<Long, Number>> results =
+ new TreeMap<String, NavigableMap<Long, Number>>();
+
+ if (result != null) {
+ NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
+ result.getMap();
+
+ NavigableMap<byte[], NavigableMap<Long, byte[]>> columnCellMap =
+ resultMap.get(columnFamilyBytes);
+
+ // could be that there is no such column family.
+ if (columnCellMap != null) {
+ for (Entry<byte[], NavigableMap<Long, byte[]>> entry : columnCellMap
+ .entrySet()) {
+ String columnName = null;
+ if (columnPrefixBytes == null) {
+ // Decode the spaces we encoded in the column name.
+ columnName = Separator.decode(entry.getKey(), Separator.SPACE);
+ } else {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ byte[][] columnNameParts =
+ Separator.QUALIFIERS.split(entry.getKey(), 2);
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ // This is the prefix that we want
+ columnName = Separator.decode(columnNameParts[1]);
+ }
+ }
+
+ // If this column has the prefix we want
+ if (columnName != null) {
+ NavigableMap<Long, Number> cellResults =
+ new TreeMap<Long, Number>();
+ NavigableMap<Long, byte[]> cells = entry.getValue();
+ if (cells != null) {
+ for (Entry<Long, byte[]> cell : cells.entrySet()) {
+ Number value =
+ (Number) GenericObjectMapper.read(cell.getValue());
+ cellResults.put(cell.getKey(), value);
+ }
+ }
+ results.put(columnName, cellResults);
+ }
+ } // for entry : columnCellMap
+ } // if columnCellMap != null
+ } // if result != null
+ return results;
+ }
+
+ /**
+ * @param result from which to read columns
+ * @param columnPrefixBytes optional prefix to limit columns. If null all
+ * columns are returned.
+ * @return the latest values of columns in the column family.
+ * @throws IOException
+ */
+ public Map<String, Object> readResults(Result result, byte[] columnPrefixBytes)
+ throws IOException {
+ Map<String, Object> results = new HashMap<String, Object>();
+
+ if (result != null) {
+ Map<byte[], byte[]> columns = result.getFamilyMap(columnFamilyBytes);
+ for (Entry<byte[], byte[]> entry : columns.entrySet()) {
+ if (entry.getKey() != null && entry.getKey().length > 0) {
+
+ String columnName = null;
+ if (columnPrefixBytes == null) {
+ // Decode the spaces we encoded in the column name.
+ columnName = Separator.decode(entry.getKey(), Separator.SPACE);
+ } else {
+ // A non-null prefix means columns are actually of the form
+ // prefix!columnNameRemainder
+ byte[][] columnNameParts =
+ Separator.QUALIFIERS.split(entry.getKey(), 2);
+ byte[] actualColumnPrefixBytes = columnNameParts[0];
+ if (Bytes.equals(columnPrefixBytes, actualColumnPrefixBytes)
+ && columnNameParts.length == 2) {
+ // This is the prefix that we want
+ columnName = Separator.decode(columnNameParts[1]);
+ }
+ }
+
+ // If this column has the prefix we want
+ if (columnName != null) {
+ Object value = GenericObjectMapper.read(entry.getValue());
+ results.put(columnName, value);
+ }
+ }
+ } // for entry
+ }
+ return results;
+ }
+
+ /**
+ * @param columnPrefixBytes The byte representation for the column prefix.
+ * Should not contain {@link Separator#QUALIFIERS}.
+ * @param qualifier for the remainder of the column. Any
+ * {@link Separator#QUALIFIERS} will be encoded in the qualifier.
+ * @return fully sanitized column qualifier that is a combination of prefix
+ * and qualifier. If prefix is null, the result is simply the encoded
+ * qualifier without any separator.
+ */
+ public static byte[] getColumnQualifier(byte[] columnPrefixBytes,
+ String qualifier) {
+
+ // We don't want column names to have spaces
+ byte[] encodedQualifier = Bytes.toBytes(Separator.SPACE.encode(qualifier));
+ if (columnPrefixBytes == null) {
+ return encodedQualifier;
+ }
+
+ // Convert qualifier to lower case, strip of separators and tag on column
+ // prefix.
+ byte[] columnQualifier =
+ Separator.QUALIFIERS.join(columnPrefixBytes, encodedQualifier);
+ return columnQualifier;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
new file mode 100644
index 0000000..2eedea0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Result;
+
+/**
+ * Used to represent a partially qualified column, where the actual column name
+ * will be composed of a prefix and the remainder of the column qualifier. The
+ * prefix can be null, in which case the column qualifier will be completely
+ * determined when the values are stored.
+ */
+public interface ColumnPrefix<T> {
+
+ /**
+ * Sends a Mutation to the table. The mutations will be buffered and sent over
+ * the wire as part of a batch.
+ *
+ * @param rowKey identifying the row to write. Nothing gets written when null.
+ * @param tableMutator used to modify the underlying HBase table. Caller is
+ * responsible to pass a mutator for the table that actually has this
+ * column.
+ * @param qualifier column qualifier. Nothing gets written when null.
+ * @param timestamp version timestamp. When null the server timestamp will be
+ * used.
+ * @param inputValue the value to write to the rowKey and column qualifier.
+ * Nothing gets written when null.
+ * @throws IOException
+ */
+ public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
+ String qualifier, Long timestamp, Object inputValue) throws IOException;
+
+ /**
+ * Get the latest version of this specified column. Note: this call clones the
+ * value content of the hosting {@link Cell}.
+ *
+ * @param result Cannot be null
+ * @param qualifier column qualifier. Nothing gets read when null.
+ * @return result object (can be cast to whatever object was written to) or
+ * null when specified column qualifier for this prefix doesn't exist
+ * in the result.
+ * @throws IOException
+ */
+ public Object readResult(Result result, String qualifier) throws IOException;
+
+ /**
+ * @param resultfrom which to read columns
+ * @return the latest values of columns in the column family with this prefix
+ * (or all of them if the prefix value is null).
+ * @throws IOException
+ */
+ public Map<String, Object> readResults(Result result) throws IOException;
+
+ /**
+ * @param result from which to reads timeseries data
+ * @return the cell values at each respective time in for form
+ * {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}
+ * @throws IOException
+ */
+ public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
+ Result result) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
new file mode 100644
index 0000000..2cb6c08
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class Range {
+ private final int startIdx;
+ private final int endIdx;
+
+ /**
+ * Defines a range from start index (inclusive) to end index (exclusive).
+ *
+ * @param start
+ * Starting index position
+ * @param end
+ * Ending index position (exclusive)
+ */
+ public Range(int start, int end) {
+ if (start < 0 || end < start) {
+ throw new IllegalArgumentException(
+ "Invalid range, required that: 0 <= start <= end; start=" + start
+ + ", end=" + end);
+ }
+
+ this.startIdx = start;
+ this.endIdx = end;
+ }
+
+ public int start() {
+ return startIdx;
+ }
+
+ public int end() {
+ return endIdx;
+ }
+
+ public int length() {
+ return endIdx - startIdx;
+ }
+}
\ No newline at end of file
[29/50] [abbrv] hadoop git commit: YARN-3051. Created storage
oriented reader interface for fetching raw entity data and made the
filesystem based implementation. Contributed by Varun Saxena.
Posted by sj...@apache.org.
YARN-3051. Created storage oriented reader interface for fetching raw entity data and made the filesystem based implementation. Contributed by Varun Saxena.
(cherry picked from commit 499ce52c7b645ec0b1cc8ac62dc9a3127b987a20)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89e6c693
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89e6c693
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89e6c693
Branch: refs/heads/YARN-2928
Commit: 89e6c6932a57cf69b1d97f8e6ac6ccd464143327
Parents: 5751abd
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Jul 6 18:11:27 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:13 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 5 +
.../storage/FileSystemTimelineReaderImpl.java | 490 ++++++++++++++++
.../timelineservice/storage/TimelineReader.java | 162 ++++++
.../TestFileSystemTimelineReaderImpl.java | 556 +++++++++++++++++++
5 files changed, 1216 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89e6c693/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a294fc5..58232b7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -73,6 +73,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3801. [JDK-8] Exclude jdk.tools from hbase-client and
hbase-testing-util (Tsuyoshi Ozawa via sjlee)
+ YARN-3051. Created storage oriented reader interface for fetching raw entity
+ data and made the filesystem based implementation. (Varun Saxena via zjshen)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89e6c693/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index a641f32..60fba85 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.util.TimelineServiceHelper;
+import org.codehaus.jackson.annotate.JsonSetter;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -335,6 +336,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("isrelatedto")
public void setIsRelatedToEntities(
Map<String, Set<String>> isRelatedToEntities) {
if (real == null) {
@@ -423,6 +425,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("relatesto")
public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
if (real == null) {
this.relatesToEntities =
@@ -441,6 +444,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("createdtime")
public void setCreatedTime(long createdTime) {
if (real == null) {
this.createdTime = createdTime;
@@ -458,6 +462,7 @@ public class TimelineEntity {
}
}
+ @JsonSetter("modifiedtime")
public void setModifiedTime(long modifiedTime) {
if (real == null) {
this.modifiedTime = modifiedTime;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89e6c693/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
new file mode 100644
index 0000000..f9f1d1d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -0,0 +1,490 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.Comparator;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.codehaus.jackson.JsonGenerationException;
+import org.codehaus.jackson.map.JsonMappingException;
+import org.codehaus.jackson.map.ObjectMapper;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * File System based implementation for TimelineReader.
+ */
+public class FileSystemTimelineReaderImpl extends AbstractService
+ implements TimelineReader {
+
+ private static final Log LOG =
+ LogFactory.getLog(FileSystemTimelineReaderImpl.class);
+
+ private String rootPath;
+ private static final String ENTITIES_DIR = "entities";
+
+ /** Default extension for output files. */
+ private static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
+
+ @VisibleForTesting
+ /** Default extension for output files. */
+ static final String APP_FLOW_MAPPING_FILE = "app_flow_mapping.csv";
+
+ @VisibleForTesting
+ /** Config param for timeline service file system storage root. */
+ static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
+
+ @VisibleForTesting
+ /** Default value for storage location on local disk. */
+ static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT =
+ "/tmp/timeline_service_data";
+
+ private final CSVFormat csvFormat =
+ CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
+
+ public FileSystemTimelineReaderImpl() {
+ super(FileSystemTimelineReaderImpl.class.getName());
+ }
+
+ @VisibleForTesting
+ String getRootPath() {
+ return rootPath;
+ }
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
+ }
+
+ /**
+ * Deserialize a POJO object from a JSON string.
+ * @param clazz
+ * class to be desirialized
+ *
+ * @param jsonString
+ * json string to deserialize
+ * @return TimelineEntity object
+ * @throws IOException
+ * @throws JsonMappingException
+ * @throws JsonGenerationException
+ */
+ public static <T> T getTimelineRecordFromJSON(
+ String jsonString, Class<T> clazz)
+ throws JsonGenerationException, JsonMappingException, IOException {
+ return mapper.readValue(jsonString, clazz);
+ }
+
+ private static void fillFields(TimelineEntity finalEntity,
+ TimelineEntity real, EnumSet<Field> fields) {
+ if (fields.contains(Field.ALL)) {
+ finalEntity.setConfigs(real.getConfigs());
+ finalEntity.setMetrics(real.getMetrics());
+ finalEntity.setInfo(real.getInfo());
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ finalEntity.setEvents(real.getEvents());
+ return;
+ }
+ for (Field field : fields) {
+ switch(field) {
+ case CONFIGS:
+ finalEntity.setConfigs(real.getConfigs());
+ break;
+ case METRICS:
+ finalEntity.setMetrics(real.getMetrics());
+ break;
+ case INFO:
+ finalEntity.setInfo(real.getInfo());
+ break;
+ case IS_RELATED_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case RELATES_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case EVENTS:
+ finalEntity.setEvents(real.getEvents());
+ break;
+ default:
+ continue;
+ }
+ }
+ }
+
+ private static boolean matchFilter(Object infoValue, Object filterValue) {
+ return infoValue.equals(filterValue);
+ }
+
+ private static boolean matchFilters(Map<String, ? extends Object> entityInfo,
+ Map<String, ? extends Object> filters) {
+ if (entityInfo == null || entityInfo.isEmpty()) {
+ return false;
+ }
+ for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
+ Object infoValue = entityInfo.get(filter.getKey());
+ if (infoValue == null) {
+ return false;
+ }
+ if (!matchFilter(infoValue, filter.getValue())) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private String getFlowRunPath(String userId, String clusterId, String flowId,
+ Long flowRunId, String appId)
+ throws IOException {
+ if (userId != null && flowId != null && flowRunId != null) {
+ return userId + "/" + flowId + "/" + flowRunId;
+ }
+ if (clusterId == null || appId == null) {
+ throw new IOException("Unable to get flow info");
+ }
+ String appFlowMappingFile = rootPath + "/" + ENTITIES_DIR + "/" +
+ clusterId + "/" + APP_FLOW_MAPPING_FILE;
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(
+ appFlowMappingFile), Charset.forName("UTF-8")));
+ CSVParser parser = new CSVParser(reader, csvFormat)) {
+ for (CSVRecord record : parser.getRecords()) {
+ if (record.size() < 4) {
+ continue;
+ }
+ String applicationId = record.get("APP");
+ if (applicationId != null && !applicationId.trim().isEmpty() &&
+ !applicationId.trim().equals(appId)) {
+ continue;
+ }
+ return record.get(1).trim() + "/" + record.get(2).trim() + "/" +
+ record.get(3).trim();
+ }
+ parser.close();
+ }
+ throw new IOException("Unable to get flow info");
+ }
+
+ private static boolean matchMetricFilters(Set<TimelineMetric> metrics,
+ Set<String> metricFilters) {
+ Set<String> tempMetrics = new HashSet<String>();
+ for (TimelineMetric metric : metrics) {
+ tempMetrics.add(metric.getId());
+ }
+
+ for (String metricFilter : metricFilters) {
+ if (!tempMetrics.contains(metricFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
+ Set<String> eventFilters) {
+ Set<String> tempEvents = new HashSet<String>();
+ for (TimelineEvent event : entityEvents) {
+ tempEvents.add(event.getId());
+ }
+
+ for (String eventFilter : eventFilters) {
+ if (!tempEvents.contains(eventFilter)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static TimelineEntity createEntityToBeReturned(TimelineEntity entity,
+ EnumSet<Field> fieldsToRetrieve) {
+ TimelineEntity entityToBeReturned = new TimelineEntity();
+ entityToBeReturned.setIdentifier(entity.getIdentifier());
+ entityToBeReturned.setCreatedTime(entity.getCreatedTime());
+ entityToBeReturned.setModifiedTime(entity.getModifiedTime());
+ if (fieldsToRetrieve != null) {
+ fillFields(entityToBeReturned, entity, fieldsToRetrieve);
+ }
+ return entityToBeReturned;
+ }
+
+ private static boolean isTimeInRange(Long time, Long timeBegin,
+ Long timeEnd) {
+ return (time >= timeBegin) && (time <= timeEnd);
+ }
+
+ private static boolean matchRelations(
+ Map<String, Set<String>> entityRelations,
+ Map<String, Set<String>> relations) {
+ for (Map.Entry<String, Set<String>> relation : relations.entrySet()) {
+ Set<String> ids = entityRelations.get(relation.getKey());
+ if (ids == null) {
+ return false;
+ }
+ for (String id : relation.getValue()) {
+ if (!ids.contains(id)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private static void mergeEntities(TimelineEntity entity1,
+ TimelineEntity entity2) {
+ // Ideally created time wont change except in the case of issue from client.
+ if (entity2.getCreatedTime() > 0) {
+ entity1.setCreatedTime(entity2.getCreatedTime());
+ }
+ if (entity2.getModifiedTime() > 0) {
+ entity1.setModifiedTime(entity2.getModifiedTime());
+ }
+ for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) {
+ entity1.addConfig(configEntry.getKey(), configEntry.getValue());
+ }
+ for (Entry<String, Object> infoEntry : entity2.getInfo().entrySet()) {
+ entity1.addInfo(infoEntry.getKey(), infoEntry.getValue());
+ }
+ for (Entry<String, Set<String>> isRelatedToEntry :
+ entity2.getIsRelatedToEntities().entrySet()) {
+ String type = isRelatedToEntry.getKey();
+ for (String entityId : isRelatedToEntry.getValue()) {
+ entity1.addIsRelatedToEntity(type, entityId);
+ }
+ }
+ for (Entry<String, Set<String>> relatesToEntry :
+ entity2.getRelatesToEntities().entrySet()) {
+ String type = relatesToEntry.getKey();
+ for (String entityId : relatesToEntry.getValue()) {
+ entity1.addRelatesToEntity(type, entityId);
+ }
+ }
+ for (TimelineEvent event : entity2.getEvents()) {
+ entity1.addEvent(event);
+ }
+ for (TimelineMetric metric2 : entity2.getMetrics()) {
+ boolean found = false;
+ for (TimelineMetric metric1 : entity1.getMetrics()) {
+ if (metric1.getId().equals(metric2.getId())) {
+ metric1.addValues(metric2.getValues());
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ entity1.addMetric(metric2);
+ }
+ }
+ }
+
+ private static TimelineEntity readEntityFromFile(BufferedReader reader)
+ throws IOException {
+ TimelineEntity entity =
+ getTimelineRecordFromJSON(reader.readLine(), TimelineEntity.class);
+ String entityStr = "";
+ while ((entityStr = reader.readLine()) != null) {
+ if (entityStr.trim().isEmpty()) {
+ continue;
+ }
+ TimelineEntity anotherEntity =
+ getTimelineRecordFromJSON(entityStr, TimelineEntity.class);
+ if (!entity.getId().equals(anotherEntity.getId()) ||
+ !entity.getType().equals(anotherEntity.getType())) {
+ continue;
+ }
+ mergeEntities(entity, anotherEntity);
+ }
+ return entity;
+ }
+
+ private Set<TimelineEntity> getEntities(File dir, String entityType,
+ Long limit, Long createdTimeBegin,
+ Long createdTimeEnd, Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ if (limit == null || limit <= 0) {
+ limit = DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null || createdTimeBegin <= 0) {
+ createdTimeBegin = 0L;
+ }
+ if (createdTimeEnd == null || createdTimeEnd <= 0) {
+ createdTimeEnd = Long.MAX_VALUE;
+ }
+ if (modifiedTimeBegin == null || modifiedTimeBegin <= 0) {
+ modifiedTimeBegin = 0L;
+ }
+ if (modifiedTimeEnd == null || modifiedTimeEnd <= 0) {
+ modifiedTimeEnd = Long.MAX_VALUE;
+ }
+
+ // First sort the selected entities based on created/start time.
+ Map<Long, Set<TimelineEntity>> sortedEntities =
+ new TreeMap<>(
+ new Comparator<Long>() {
+ @Override
+ public int compare(Long l1, Long l2) {
+ return l2.compareTo(l1);
+ }
+ }
+ );
+ for (File entityFile : dir.listFiles()) {
+ if (!entityFile.getName().contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
+ continue;
+ }
+ try (BufferedReader reader =
+ new BufferedReader(
+ new InputStreamReader(
+ new FileInputStream(
+ entityFile), Charset.forName("UTF-8")))) {
+ TimelineEntity entity = readEntityFromFile(reader);
+ if (!entity.getType().equals(entityType)) {
+ continue;
+ }
+ if (!isTimeInRange(entity.getCreatedTime(), createdTimeBegin,
+ createdTimeEnd)) {
+ continue;
+ }
+ if (!isTimeInRange(entity.getModifiedTime(), modifiedTimeBegin,
+ modifiedTimeEnd)) {
+ continue;
+ }
+ if (relatesTo != null && !relatesTo.isEmpty() &&
+ !matchRelations(entity.getRelatesToEntities(), relatesTo)) {
+ continue;
+ }
+ if (isRelatedTo != null && !isRelatedTo.isEmpty() &&
+ !matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
+ continue;
+ }
+ if (infoFilters != null && !infoFilters.isEmpty() &&
+ !matchFilters(entity.getInfo(), infoFilters)) {
+ continue;
+ }
+ if (configFilters != null && !configFilters.isEmpty() &&
+ !matchFilters(entity.getConfigs(), configFilters)) {
+ continue;
+ }
+ if (metricFilters != null && !metricFilters.isEmpty() &&
+ !matchMetricFilters(entity.getMetrics(), metricFilters)) {
+ continue;
+ }
+ if (eventFilters != null && !eventFilters.isEmpty() &&
+ !matchEventFilters(entity.getEvents(), eventFilters)) {
+ continue;
+ }
+ TimelineEntity entityToBeReturned =
+ createEntityToBeReturned(entity, fieldsToRetrieve);
+ Set<TimelineEntity> entitiesCreatedAtSameTime =
+ sortedEntities.get(entityToBeReturned.getCreatedTime());
+ if (entitiesCreatedAtSameTime == null) {
+ entitiesCreatedAtSameTime = new HashSet<TimelineEntity>();
+ }
+ entitiesCreatedAtSameTime.add(entityToBeReturned);
+ sortedEntities.put(
+ entityToBeReturned.getCreatedTime(), entitiesCreatedAtSameTime);
+ }
+ }
+
+ Set<TimelineEntity> entities = new HashSet<TimelineEntity>();
+ long entitiesAdded = 0;
+ for (Set<TimelineEntity> entitySet : sortedEntities.values()) {
+ for (TimelineEntity entity : entitySet) {
+ entities.add(entity);
+ ++entitiesAdded;
+ if (entitiesAdded >= limit) {
+ return entities;
+ }
+ }
+ }
+ return entities;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ rootPath = conf.get(TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ public TimelineEntity getEntity(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) throws IOException {
+ String flowRunPath = getFlowRunPath(userId, clusterId, flowId,
+ flowRunId, appId);
+ File dir = new File(new File(rootPath, ENTITIES_DIR),
+ clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
+ File entityFile =
+ new File(dir, entityId + TIMELINE_SERVICE_STORAGE_EXTENSION);
+ try (BufferedReader reader =
+ new BufferedReader(new InputStreamReader(
+ new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
+ TimelineEntity entity = readEntityFromFile(reader);
+ return createEntityToBeReturned(entity, fieldsToRetrieve);
+ }
+ }
+
+ @Override
+ public Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException {
+ String flowRunPath =
+ getFlowRunPath(userId, clusterId, flowId, flowRunId, appId);
+ File dir =
+ new File(new File(rootPath, ENTITIES_DIR),
+ clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
+ return getEntities(dir, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, fieldsToRetrieve);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89e6c693/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
new file mode 100644
index 0000000..e4e305e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.IOException;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+
+/** ATSv2 reader interface. */
+@Private
+@Unstable
+public interface TimelineReader extends Service {
+
+ /**
+ * Default limit for {@link #getEntities}.
+ */
+ long DEFAULT_LIMIT = 100;
+
+ /**
+ * Possible fields to retrieve for {@link #getEntities} and
+ * {@link #getEntity}.
+ */
+ public enum Field {
+ ALL,
+ EVENTS,
+ INFO,
+ METRICS,
+ CONFIGS,
+ RELATES_TO,
+ IS_RELATED_TO
+ }
+
+ /**
+ * <p>The API to fetch the single entity given the entity identifier in the
+ * scope of the given context.</p>
+ *
+ * @param userId
+ * Context user Id(optional).
+ * @param clusterId
+ * Context cluster Id(mandatory).
+ * @param flowId
+ * Context flow Id (optional).
+ * @param flowRunId
+ * Context flow run Id (optional).
+ * @param appId
+ * Context app Id (mandatory)
+ * @param entityType
+ * Entity type (mandatory)
+ * @param entityId
+ * Entity Id (mandatory)
+ * @param fieldsToRetrieve
+ * Specifies which fields of the entity object to retrieve(optional), see
+ * {@link Field}. If null, retrieves 4 fields namely entity id,
+ * entity type, entity created time and entity modified time. All
+ * entities will be returned if {@link Field#ALL} is specified.
+ * @return a {@link TimelineEntity} instance or null. The entity will
+ * contain the metadata plus the given fields to retrieve.
+ * @throws IOException
+ */
+ TimelineEntity getEntity(String userId, String clusterId, String flowId,
+ Long flowRunId, String appId, String entityType, String entityId,
+ EnumSet<Field> fieldsToRetrieve) throws IOException;
+
+ /**
+ * <p>The API to search for a set of entities of the given the entity type in
+ * the scope of the given context which matches the given predicates. The
+ * predicates include the created/modified time window, limit to number of
+ * entities to be returned, and the entities can be filtered by checking
+ * whether they contain the given info/configs entries in the form of
+ * key/value pairs, given metrics in the form of metricsIds and its relation
+ * with metric values given events in the form of the Ids, and whether they
+ * relate to/are related to other entities. For those parameters which have
+ * multiple entries, the qualified entity needs to meet all or them.</p>
+ *
+ * @param userId
+ * Context user Id(optional).
+ * @param clusterId
+ * Context cluster Id(mandatory).
+ * @param flowId
+ * Context flow Id (optional).
+ * @param flowRunId
+ * Context flow run Id (optional).
+ * @param appId
+ * Context app Id (mandatory)
+ * @param entityType
+ * Entity type (mandatory)
+ * @param limit
+ * A limit on the number of entities to return (optional). If null or <=0,
+ * defaults to {@link #DEFAULT_LIMIT}.
+ * @param createdTimeBegin
+ * Matched entities should not be created before this timestamp (optional).
+ * If null or <=0, defaults to 0.
+ * @param createdTimeEnd
+ * Matched entities should not be created after this timestamp (optional).
+ * If null or <=0, defaults to {@link Long#MAX_VALUE}.
+ * @param modifiedTimeBegin
+ * Matched entities should not be modified before this timestamp
+ * (optional). If null or <=0, defaults to 0.
+ * @param modifiedTimeEnd
+ * Matched entities should not be modified after this timestamp (optional).
+ * If null or <=0, defaults to {@link Long#MAX_VALUE}.
+ * @param relatesTo
+ * Matched entities should relate to given entities (optional).
+ * @param isRelatedTo
+ * Matched entities should be related to given entities (optional).
+ * @param infoFilters
+ * Matched entities should have exact matches to the given info represented
+ * as key-value pairs (optional). If null or empty, the filter is not
+ * applied.
+ * @param configFilters
+ * Matched entities should have exact matches to the given configs
+ * represented as key-value pairs (optional). If null or empty, the filter
+ * is not applied.
+ * @param metricFilters
+ * Matched entities should contain the given metrics (optional). If null
+ * or empty, the filter is not applied.
+ * @param eventFilters
+ * Matched entities should contain the given events (optional). If null
+ * or empty, the filter is not applied.
+ * @param fieldsToRetrieve
+ * Specifies which fields of the entity object to retrieve(optional), see
+ * {@link Field}. If null, retrieves 4 fields namely entity id,
+ * entity type, entity created time and entity modified time. All
+ * entities will be returned if {@link Field#ALL} is specified.
+ * @return A set of {@link TimelineEntity} instances of the given entity type
+ * in the given context scope which matches the given predicates
+ * ordered by created time, descending. Each entity will only contain the
+ * metadata(id, type, created and modified times) plus the given fields to
+ * retrieve.
+ * @throws IOException
+ */
+ Set<TimelineEntity> getEntities(String userId, String clusterId,
+ String flowId, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89e6c693/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
new file mode 100644
index 0000000..4e23e49
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -0,0 +1,556 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVPrinter;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestFileSystemTimelineReaderImpl {
+
+ private static final String rootDir =
+ FileSystemTimelineReaderImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT;
+ FileSystemTimelineReaderImpl reader;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ loadEntityData();
+ // Create app flow mapping file.
+ CSVFormat format =
+ CSVFormat.DEFAULT.withHeader("APP", "USER", "FLOW", "FLOWRUN");
+ String appFlowMappingFile = rootDir + "/entities/cluster1/" +
+ FileSystemTimelineReaderImpl.APP_FLOW_MAPPING_FILE;
+ try (PrintWriter out =
+ new PrintWriter(new BufferedWriter(
+ new FileWriter(appFlowMappingFile, true)));
+ CSVPrinter printer = new CSVPrinter(out, format)){
+ printer.printRecord("app1", "user1", "flow1", 1);
+ printer.printRecord("app2","user1","flow1,flow",1);
+ printer.close();
+ }
+ (new File(rootDir)).deleteOnExit();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ FileUtils.deleteDirectory(new File(rootDir));
+ }
+
+ @Before
+ public void init() throws Exception {
+ reader = new FileSystemTimelineReaderImpl();
+ Configuration conf = new YarnConfiguration();
+ conf.set(FileSystemTimelineReaderImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ rootDir);
+ reader.init(conf);
+ }
+
+ private static void writeEntityFile(TimelineEntity entity, File dir)
+ throws Exception {
+ if (!dir.exists()) {
+ if (!dir.mkdirs()) {
+ throw new IOException("Could not create directories for " + dir);
+ }
+ }
+ String fileName = dir.getAbsolutePath() + "/" + entity.getId() + ".thist";
+ try (PrintWriter out =
+ new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)))){
+ out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ out.write("\n");
+ out.close();
+ }
+ }
+
+ private static void loadEntityData() throws Exception {
+ File appDir = new File(rootDir +
+ "/entities/cluster1/user1/flow1/1/app1/app/");
+ TimelineEntity entity11 = new TimelineEntity();
+ entity11.setId("id_1");
+ entity11.setType("app");
+ entity11.setCreatedTime(1425016502000L);
+ entity11.setModifiedTime(1425016502050L);
+ Map<String, Object> info1 = new HashMap<String, Object>();
+ info1.put("info1", "val1");
+ entity11.addInfo(info1);
+ TimelineEvent event = new TimelineEvent();
+ event.setId("event_1");
+ event.setTimestamp(1425016502003L);
+ entity11.addEvent(event);
+ Set<TimelineMetric> metrics = new HashSet<TimelineMetric>();
+ TimelineMetric metric1 = new TimelineMetric();
+ metric1.setId("metric1");
+ metric1.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric1.addValue(1425016502006L, 113.2F);
+ metrics.add(metric1);
+ TimelineMetric metric2 = new TimelineMetric();
+ metric2.setId("metric2");
+ metric2.setType(TimelineMetric.Type.TIME_SERIES);
+ metric2.addValue(1425016502016L, 34);
+ metrics.add(metric2);
+ entity11.setMetrics(metrics);
+ Map<String,String> configs = new HashMap<String, String>();
+ configs.put("config_1", "123");
+ entity11.setConfigs(configs);
+ entity11.addRelatesToEntity("flow", "flow1");
+ entity11.addIsRelatedToEntity("type1", "tid1_1");
+ writeEntityFile(entity11, appDir);
+ TimelineEntity entity12 = new TimelineEntity();
+ entity12.setId("id_1");
+ entity12.setType("app");
+ entity12.setModifiedTime(1425016503000L);
+ configs.clear();
+ configs.put("config_2", "23");
+ configs.put("config_3", "abc");
+ entity12.addConfigs(configs);
+ metrics.clear();
+ TimelineMetric metric12 = new TimelineMetric();
+ metric12.setId("metric2");
+ metric12.setType(TimelineMetric.Type.TIME_SERIES);
+ metric12.addValue(1425016502032L, 48);
+ metric12.addValue(1425016502054L, 51);
+ metrics.add(metric12);
+ TimelineMetric metric3 = new TimelineMetric();
+ metric3.setId("metric3");
+ metric3.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric3.addValue(1425016502060L, 23L);
+ metrics.add(metric3);
+ entity12.setMetrics(metrics);
+ entity12.addIsRelatedToEntity("type1", "tid1_2");
+ entity12.addIsRelatedToEntity("type2", "tid2_1`");
+ TimelineEvent event15 = new TimelineEvent();
+ event15.setId("event_5");
+ event15.setTimestamp(1425016502017L);
+ entity12.addEvent(event15);
+ writeEntityFile(entity12, appDir);
+
+ TimelineEntity entity2 = new TimelineEntity();
+ entity2.setId("id_2");
+ entity2.setType("app");
+ entity2.setCreatedTime(1425016501050L);
+ entity2.setModifiedTime(1425016502010L);
+ Map<String, Object> info2 = new HashMap<String, Object>();
+ info1.put("info2", 4);
+ entity2.addInfo(info2);
+ Map<String,String> configs2 = new HashMap<String, String>();
+ configs2.put("config_1", "123");
+ configs2.put("config_3", "def");
+ entity2.setConfigs(configs2);
+ TimelineEvent event2 = new TimelineEvent();
+ event2.setId("event_2");
+ event2.setTimestamp(1425016501003L);
+ entity2.addEvent(event2);
+ Set<TimelineMetric> metrics2 = new HashSet<TimelineMetric>();
+ TimelineMetric metric21 = new TimelineMetric();
+ metric21.setId("metric1");
+ metric21.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric21.addValue(1425016501006L, 123.2F);
+ metrics2.add(metric21);
+ TimelineMetric metric22 = new TimelineMetric();
+ metric22.setId("metric2");
+ metric22.setType(TimelineMetric.Type.TIME_SERIES);
+ metric22.addValue(1425016501056L, 31);
+ metric22.addValue(1425016501084L, 70);
+ metrics2.add(metric22);
+ TimelineMetric metric23 = new TimelineMetric();
+ metric23.setId("metric3");
+ metric23.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric23.addValue(1425016502060L, 23L);
+ metrics2.add(metric23);
+ entity2.setMetrics(metrics2);
+ entity2.addRelatesToEntity("flow", "flow2");
+ writeEntityFile(entity2, appDir);
+
+ TimelineEntity entity3 = new TimelineEntity();
+ entity3.setId("id_3");
+ entity3.setType("app");
+ entity3.setCreatedTime(1425016501050L);
+ entity3.setModifiedTime(1425016502010L);
+ Map<String, Object> info3 = new HashMap<String, Object>();
+ info3.put("info2", 3.5);
+ entity3.addInfo(info3);
+ Map<String,String> configs3 = new HashMap<String, String>();
+ configs3.put("config_1", "123");
+ configs3.put("config_3", "abc");
+ entity3.setConfigs(configs3);
+ TimelineEvent event3 = new TimelineEvent();
+ event3.setId("event_2");
+ event3.setTimestamp(1425016501003L);
+ entity3.addEvent(event3);
+ TimelineEvent event4 = new TimelineEvent();
+ event4.setId("event_4");
+ event4.setTimestamp(1425016502006L);
+ entity3.addEvent(event4);
+ Set<TimelineMetric> metrics3 = new HashSet<TimelineMetric>();
+ TimelineMetric metric31 = new TimelineMetric();
+ metric31.setId("metric1");
+ metric31.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric31.addValue(1425016501006L, 124.8F);
+ metrics3.add(metric31);
+ TimelineMetric metric32 = new TimelineMetric();
+ metric32.setId("metric2");
+ metric32.setType(TimelineMetric.Type.TIME_SERIES);
+ metric32.addValue(1425016501056L, 31);
+ metric32.addValue(1425016501084L, 74);
+ metrics3.add(metric32);
+ entity3.setMetrics(metrics3);
+ entity3.addIsRelatedToEntity("type1", "tid1_2");
+ writeEntityFile(entity3, appDir);
+
+ TimelineEntity entity4 = new TimelineEntity();
+ entity4.setId("id_4");
+ entity4.setType("app");
+ entity4.setCreatedTime(1425016502050L);
+ entity4.setModifiedTime(1425016503010L);
+ TimelineEvent event44 = new TimelineEvent();
+ event44.setId("event_4");
+ event44.setTimestamp(1425016502003L);
+ entity4.addEvent(event44);
+ writeEntityFile(entity4, appDir);
+
+ File appDir2 = new File(rootDir +
+ "/entities/cluster1/user1/flow1,flow/1/app2/app/");
+ TimelineEntity entity5 = new TimelineEntity();
+ entity5.setId("id_5");
+ entity5.setType("app");
+ entity5.setCreatedTime(1425016502050L);
+ entity5.setModifiedTime(1425016503010L);
+ writeEntityFile(entity5, appDir2);
+ }
+
+ public TimelineReader getTimelineReader() {
+ return reader;
+ }
+
+ @Test
+ public void testGetEntityDefaultView() throws Exception {
+ // If no fields are specified, entity is returned with default view i.e.
+ // only the id, created and modified time
+ TimelineEntity result =
+ reader.getEntity("user1", "cluster1", "flow1", 1L, "app1",
+ "app", "id_1", null);
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(0, result.getConfigs().size());
+ Assert.assertEquals(0, result.getMetrics().size());
+ }
+
+ @Test
+ public void testGetEntityByClusterAndApp() throws Exception {
+ // Cluster and AppId should be enough to get an entity.
+ TimelineEntity result =
+ reader.getEntity(null, "cluster1", null, null, "app1",
+ "app", "id_1", null);
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(0, result.getConfigs().size());
+ Assert.assertEquals(0, result.getMetrics().size());
+ }
+
+ /** This test checks whether we can handle commas in app flow mapping csv */
+ @Test
+ public void testAppFlowMappingCsv() throws Exception {
+ // Test getting an entity by cluster and app where flow entry
+ // in app flow mapping csv has commas.
+ TimelineEntity result =
+ reader.getEntity(null, "cluster1", null, null, "app2",
+ "app", "id_5", null);
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_5")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502050L, result.getCreatedTime());
+ Assert.assertEquals(1425016503010L, result.getModifiedTime());
+ }
+
+ @Test
+ public void testGetEntityCustomFields() throws Exception {
+ // Specified fields in addition to default view will be returned.
+ TimelineEntity result =
+ reader.getEntity("user1", "cluster1", "flow1", 1L,
+ "app1", "app", "id_1",
+ EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS));
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(3, result.getConfigs().size());
+ Assert.assertEquals(3, result.getMetrics().size());
+ Assert.assertEquals(1, result.getInfo().size());
+ // No events will be returned
+ Assert.assertEquals(0, result.getEvents().size());
+ }
+
+ @Test
+ public void testGetEntityAllFields() throws Exception {
+ // All fields of TimelineEntity will be returned.
+ TimelineEntity result =
+ reader.getEntity("user1", "cluster1", "flow1", 1L,
+ "app1", "app", "id_1", EnumSet.of(Field.ALL));
+ Assert.assertEquals(
+ (new TimelineEntity.Identifier("app", "id_1")).toString(),
+ result.getIdentifier().toString());
+ Assert.assertEquals(1425016502000L, result.getCreatedTime());
+ Assert.assertEquals(1425016503000L, result.getModifiedTime());
+ Assert.assertEquals(3, result.getConfigs().size());
+ Assert.assertEquals(3, result.getMetrics().size());
+ // All fields including events will be returned.
+ Assert.assertEquals(2, result.getEvents().size());
+ }
+
+ @Test
+ public void testGetAllEntities() throws Exception {
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, null, null,
+ null, null);
+ // All 3 entities will be returned
+ Assert.assertEquals(4, result.size());
+ }
+
+ @Test
+ public void testGetEntitiesWithLimit() throws Exception {
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ 2L, null, null, null, null, null, null, null, null, null,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ // Needs to be rewritten once hashcode and equals for
+ // TimelineEntity is implemented
+ // Entities with id_1 and id_4 should be returned,
+ // based on created time, descending.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_4")) {
+ Assert.fail("Entity not sorted by created time");
+ }
+ }
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ 3L, null, null, null, null, null, null, null, null, null,
+ null, null);
+ // Even though 2 entities out of 4 have same created time, one entity
+ // is left out due to limit
+ Assert.assertEquals(3, result.size());
+ }
+
+ @Test
+ public void testGetEntitiesByTimeWindows() throws Exception {
+ // Get entities based on created time start and end time range.
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, 1425016502030L, 1425016502060L, null, null, null, null, null,
+ null, null, null, null);
+ Assert.assertEquals(1, result.size());
+ // Only one entity with ID id_4 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on created time range");
+ }
+ }
+
+ // Get entities if only created time end is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, 1425016502010L, null, null, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(3, result.size());
+ for (TimelineEntity entity : result) {
+ if (entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on created time range");
+ }
+ }
+
+ // Get entities if only created time start is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, 1425016502010L, null, null, null, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on created time range");
+ }
+ }
+
+ // Get entities based on modified time start and end time range.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, 1425016502090L, 1425016503020L, null, null, null,
+ null, null, null, null);
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_1 and id_4 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on modified time range");
+ }
+ }
+
+ // Get entities if only modified time end is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, 1425016502090L, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (entity.getId().equals("id_1") || entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on modified time range");
+ }
+ }
+
+ // Get entities if only modified time start is specified.
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, 1425016503005L, null, null, null, null, null,
+ null, null, null);
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_4")) {
+ Assert.fail("Incorrect filtering based on modified time range");
+ }
+ }
+ }
+
+ @Test
+ public void testGetFilteredEntities() throws Exception {
+ // Get entities based on info filters.
+ Map<String, Object> infoFilters = new HashMap<String, Object>();
+ infoFilters.put("info2", 3.5);
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, infoFilters, null, null,
+ null, null);
+ Assert.assertEquals(1, result.size());
+ // Only one entity with ID id_3 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on info filters");
+ }
+ }
+
+ // Get entities based on config filters.
+ Map<String, String> configFilters = new HashMap<String, String>();
+ configFilters.put("config_1", "123");
+ configFilters.put("config_3", "abc");
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, configFilters, null,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on config filters");
+ }
+ }
+
+ // Get entities based on event filters.
+ Set<String> eventFilters = new HashSet<String>();
+ eventFilters.add("event_2");
+ eventFilters.add("event_4");
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, null, null,
+ eventFilters, null);
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on event filters");
+ }
+ }
+
+ // Get entities based on metric filters.
+ Set<String> metricFilters = new HashSet<String>();
+ metricFilters.add("metric3");
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, null, null, null, metricFilters,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_1 and id_2 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on metric filters");
+ }
+ }
+ }
+
+ @Test
+ public void testGetEntitiesByRelations() throws Exception {
+ // Get entities based on relatesTo.
+ Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
+ Set<String> relatesToIds = new HashSet<String>();
+ relatesToIds.add("flow1");
+ relatesTo.put("flow", relatesToIds);
+ Set<TimelineEntity> result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, relatesTo, null, null, null, null,
+ null, null);
+ Assert.assertEquals(1, result.size());
+ // Only one entity with ID id_1 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1")) {
+ Assert.fail("Incorrect filtering based on relatesTo");
+ }
+ }
+
+ // Get entities based on isRelatedTo.
+ Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
+ Set<String> isRelatedToIds = new HashSet<String>();
+ isRelatedToIds.add("tid1_2");
+ isRelatedTo.put("type1", isRelatedToIds);
+ result =
+ reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
+ null, null, null, null, null, null, isRelatedTo, null, null, null,
+ null, null);
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_1 and id_3 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on isRelatedTo");
+ }
+ }
+ }
+}
[27/50] [abbrv] hadoop git commit: YARN-3276. Code cleanup for
timeline service API records. Contributed by Junping Du.
Posted by sj...@apache.org.
YARN-3276. Code cleanup for timeline service API records. Contributed by Junping Du.
(cherry picked from commit d88f30ba5359f59fb71b93a55e1c1d9a1c0dff8e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6dd2397e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6dd2397e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6dd2397e
Branch: refs/heads/YARN-2928
Commit: 6dd2397e520f18a1bb84d6dc1f19e8762e569040
Parents: 534aae5
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Jun 3 15:13:29 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:12 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../api/records/timeline/TimelineEntity.java | 21 ++----
.../api/records/timeline/TimelineEvent.java | 8 +--
.../records/timelineservice/TimelineEntity.java | 33 ++-------
.../records/timelineservice/TimelineEvent.java | 7 +-
.../records/timelineservice/TimelineMetric.java | 2 +-
.../hadoop/yarn/util/TimelineServiceHelper.java | 47 ++++++++++++
.../impl/pb/AllocateResponsePBImpl.java | 4 +-
.../yarn/util/TestTimelineServiceHelper.java | 76 ++++++++++++++++++++
9 files changed, 147 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5d8aefc..b3e8738 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -84,6 +84,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
IMPROVEMENTS
+ YARN-3276. Code cleanup for timeline service API records. (Junping Du via
+ zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
index a43259b..e695050 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEntity.java
@@ -34,6 +34,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
/**
* <p>
@@ -231,11 +232,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
*/
public void setRelatedEntities(
Map<String, Set<String>> relatedEntities) {
- if (relatedEntities != null && !(relatedEntities instanceof HashMap)) {
- this.relatedEntities = new HashMap<String, Set<String>>(relatedEntities);
- } else {
- this.relatedEntities = (HashMap<String, Set<String>>) relatedEntities;
- }
+ this.relatedEntities = TimelineServiceHelper.mapCastToHashMap(
+ relatedEntities);
}
/**
@@ -297,11 +295,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
* a map of primary filters
*/
public void setPrimaryFilters(Map<String, Set<Object>> primaryFilters) {
- if (primaryFilters != null && !(primaryFilters instanceof HashMap)) {
- this.primaryFilters = new HashMap<String, Set<Object>>(primaryFilters);
- } else {
- this.primaryFilters = (HashMap<String, Set<Object>>) primaryFilters;
- }
+ this.primaryFilters =
+ TimelineServiceHelper.mapCastToHashMap(primaryFilters);
}
/**
@@ -350,11 +345,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
* a map of other information
*/
public void setOtherInfo(Map<String, Object> otherInfo) {
- if (otherInfo != null && !(otherInfo instanceof HashMap)) {
- this.otherInfo = new HashMap<String, Object>(otherInfo);
- } else {
- this.otherInfo = (HashMap<String, Object>) otherInfo;
- }
+ this.otherInfo = TimelineServiceHelper.mapCastToHashMap(otherInfo);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
index 73b2e72..d5611f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timeline/TimelineEvent.java
@@ -29,6 +29,7 @@ import javax.xml.bind.annotation.XmlRootElement;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
/**
* The class that contains the information of an event that is related to some
@@ -135,11 +136,8 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
* a map of of the information of the event
*/
public void setEventInfo(Map<String, Object> eventInfo) {
- if (eventInfo != null && !(eventInfo instanceof HashMap)) {
- this.eventInfo = new HashMap<String, Object>(eventInfo);
- } else {
- this.eventInfo = (HashMap<String, Object>) eventInfo;
- }
+ this.eventInfo = TimelineServiceHelper.mapCastToHashMap(
+ eventInfo);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 3be7f52..defadec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -184,11 +185,7 @@ public class TimelineEntity {
public void setInfo(Map<String, Object> info) {
if (real == null) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
+ this.info = TimelineServiceHelper.mapCastToHashMap(info);
} else {
real.setInfo(info);
}
@@ -231,11 +228,7 @@ public class TimelineEntity {
public void setConfigs(Map<String, String> configs) {
if (real == null) {
- if (configs != null && !(configs instanceof HashMap)) {
- this.configs = new HashMap<String, String>(configs);
- } else {
- this.configs = (HashMap<String, String>) configs;
- }
+ this.configs = TimelineServiceHelper.mapCastToHashMap(configs);
} else {
real.setConfigs(configs);
}
@@ -345,14 +338,8 @@ public class TimelineEntity {
public void setIsRelatedToEntities(
Map<String, Set<String>> isRelatedToEntities) {
if (real == null) {
- if (isRelatedToEntities != null &&
- !(isRelatedToEntities instanceof HashMap)) {
- this.isRelatedToEntities =
- new HashMap<String, Set<String>>(isRelatedToEntities);
- } else {
- this.isRelatedToEntities =
- (HashMap<String, Set<String>>) isRelatedToEntities;
- }
+ this.isRelatedToEntities =
+ TimelineServiceHelper.mapCastToHashMap(isRelatedToEntities);
} else {
real.setIsRelatedToEntities(isRelatedToEntities);
}
@@ -438,14 +425,8 @@ public class TimelineEntity {
public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
if (real == null) {
- if (relatesToEntities != null &&
- !(relatesToEntities instanceof HashMap)) {
- this.relatesToEntities =
- new HashMap<String, Set<String>>(relatesToEntities);
- } else {
- this.relatesToEntities =
- (HashMap<String, Set<String>>) relatesToEntities;
- }
+ this.relatesToEntities =
+ TimelineServiceHelper.mapCastToHashMap(relatesToEntities);
} else {
real.setRelatesToEntities(relatesToEntities);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index 517c88c..b6b82a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
@@ -61,11 +62,7 @@ public class TimelineEvent {
}
public void setInfo(Map<String, Object> info) {
- if (info != null && !(info instanceof HashMap)) {
- this.info = new HashMap<String, Object>(info);
- } else {
- this.info = (HashMap<String, Object>) info;
- }
+ this.info = TimelineServiceHelper.mapCastToHashMap(info);
}
public void addInfo(Map<String, Object> info) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index c897d39..172b56c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -44,7 +44,7 @@ public class TimelineMetric {
private Comparator<Long> reverseComparator = new Comparator<Long>() {
@Override
public int compare(Long l1, Long l2) {
- return -l1.compareTo(l2);
+ return l2.compareTo(l1);
}
};
private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java
new file mode 100644
index 0000000..ff6ebbd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/TimelineServiceHelper.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+
+/**
+ * Helper class for Timeline service.
+ */
+@LimitedPrivate({ "MapReduce", "YARN" })
+public final class TimelineServiceHelper {
+
+ private TimelineServiceHelper() {
+ // Utility classes should not have a public or default constructor.
+ }
+
+ /**
+ * Cast map to HashMap for generic type.
+ * @param originalMap the map need to be casted
+ * @return casted HashMap object
+ */
+ public static <E, V> HashMap<E, V> mapCastToHashMap(
+ Map<E, V> originalMap) {
+ return originalMap == null ? null : originalMap instanceof HashMap ?
+ (HashMap<E, V>) originalMap : new HashMap<E, V>(originalMap);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
index 8dc473c..6cd5b75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
@@ -387,13 +387,13 @@ public class AllocateResponsePBImpl extends AllocateResponse {
@Override
- public String getCollectorAddr() {
+ public synchronized String getCollectorAddr() {
AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
return p.getCollectorAddr();
}
@Override
- public void setCollectorAddr(String collectorAddr) {
+ public synchronized void setCollectorAddr(String collectorAddr) {
maybeInitBuilder();
if (collectorAddr == null) {
builder.clearCollectorAddr();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dd2397e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
new file mode 100644
index 0000000..f852df0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestTimelineServiceHelper.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.util;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestTimelineServiceHelper {
+
+ @Test
+ public void testMapCastToHashMap() {
+
+ // Test null map be casted to null
+ Map<String, String> nullMap = null;
+ Assert.assertNull(TimelineServiceHelper.mapCastToHashMap(nullMap));
+
+ // Test empty hashmap be casted to a empty hashmap
+ Map<String, String> emptyHashMap = new HashMap<String, String>();
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyHashMap).size(), 0);
+
+ // Test empty non-hashmap be casted to a empty hashmap
+ Map<String, String> emptyTreeMap = new TreeMap<String, String>();
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(emptyTreeMap).size(), 0);
+
+ // Test non-empty hashmap be casted to hashmap correctly
+ Map<String, String> firstHashMap = new HashMap<String, String>();
+ String key = "KEY";
+ String value = "VALUE";
+ firstHashMap.put(key, value);
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(firstHashMap), firstHashMap);
+
+ // Test non-empty non-hashmap is casted correctly.
+ Map<String, String> firstTreeMap = new TreeMap<String, String>();
+ firstTreeMap.put(key, value);
+ HashMap<String, String> alternateHashMap =
+ TimelineServiceHelper.mapCastToHashMap(firstTreeMap);
+ Assert.assertEquals(firstTreeMap.size(), alternateHashMap.size());
+ Assert.assertEquals(alternateHashMap.get(key), value);
+
+ // Test complicated hashmap be casted correctly
+ Map<String, Set<String>> complicatedHashMap = new HashMap<String, Set<String>>();
+ Set<String> hashSet = new HashSet<String>();
+ hashSet.add(value);
+ complicatedHashMap.put(key, hashSet);
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedHashMap),
+ complicatedHashMap);
+
+ // Test complicated non-hashmap get casted correctly
+ Map<String, Set<String>> complicatedTreeMap = new TreeMap<String, Set<String>>();
+ complicatedTreeMap.put(key, hashSet);
+ Assert.assertEquals(TimelineServiceHelper.mapCastToHashMap(complicatedTreeMap).get(key),
+ hashSet);
+ }
+
+}
[38/50] [abbrv] hadoop git commit: YARN-3906. Split the application
table from the entity table. Contributed by Sangjin Lee.
Posted by sj...@apache.org.
YARN-3906. Split the application table from the entity table. Contributed by Sangjin Lee.
(cherry picked from commit bcd755eba9466ce277d3c14192c31da6462c4ab3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51029a75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51029a75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51029a75
Branch: refs/heads/YARN-2928
Commit: 51029a751236b1cff4c92bab5612395c76f07d8e
Parents: 4bdf34f
Author: Junping Du <ju...@apache.org>
Authored: Tue Aug 11 16:59:21 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/HBaseTimelineReaderImpl.java | 202 ++++++++----
.../storage/HBaseTimelineWriterImpl.java | 145 ++++++---
.../storage/TimelineSchemaCreator.java | 10 +
.../storage/application/ApplicationColumn.java | 136 ++++++++
.../application/ApplicationColumnFamily.java | 65 ++++
.../application/ApplicationColumnPrefix.java | 217 +++++++++++++
.../storage/application/ApplicationRowKey.java | 67 ++++
.../storage/application/ApplicationTable.java | 164 ++++++++++
.../storage/application/package-info.java | 25 ++
.../storage/entity/EntityColumnPrefix.java | 2 +-
.../storage/entity/EntityTable.java | 59 ++--
.../storage/TestHBaseTimelineWriterImpl.java | 321 ++++++++++++++++---
13 files changed, 1230 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2e06793..a4aeef8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -85,6 +85,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3049. [Storage Implementation] Implement storage reader interface to
fetch raw data from HBase backend (Zhijie Shen via sjlee)
+ YARN-3906. Split the application table from the entity table. (Sangjin Lee
+ via junping_du)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 5258b9c..094f868 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -18,7 +18,19 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
-import com.google.common.base.Preconditions;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.Collection;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -32,11 +44,17 @@ import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineReaderUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
@@ -45,18 +63,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-import java.io.IOException;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
+import com.google.common.base.Preconditions;
public class HBaseTimelineReaderImpl
extends AbstractService implements TimelineReader {
@@ -70,6 +77,7 @@ public class HBaseTimelineReaderImpl
private Connection conn;
private EntityTable entityTable;
private AppToFlowTable appToFlowTable;
+ private ApplicationTable applicationTable;
public HBaseTimelineReaderImpl() {
super(HBaseTimelineReaderImpl.class.getName());
@@ -82,6 +90,7 @@ public class HBaseTimelineReaderImpl
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable();
appToFlowTable = new AppToFlowTable();
+ applicationTable = new ApplicationTable();
}
@Override
@@ -109,14 +118,24 @@ public class HBaseTimelineReaderImpl
fieldsToRetrieve = EnumSet.noneOf(Field.class);
}
- byte[] rowKey = EntityRowKey.getRowKey(
- clusterId, userId, flowId, flowRunId, appId, entityType, entityId);
+ boolean isApplication = isApplicationEntity(entityType);
+ byte[] rowKey = isApplication ?
+ ApplicationRowKey.getRowKey(clusterId, userId, flowId, flowRunId,
+ appId) :
+ EntityRowKey.getRowKey(clusterId, userId, flowId, flowRunId, appId,
+ entityType, entityId);
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
- return parseEntity(
- entityTable.getResult(hbaseConf, conn, get), fieldsToRetrieve,
+ Result result = isApplication ?
+ applicationTable.getResult(hbaseConf, conn, get) :
+ entityTable.getResult(hbaseConf, conn, get);
+ return parseEntity(result, fieldsToRetrieve,
false, DEFAULT_BEGIN_TIME, DEFAULT_END_TIME, false, DEFAULT_BEGIN_TIME,
- DEFAULT_END_TIME, null, null, null, null, null, null);
+ DEFAULT_END_TIME, null, null, null, null, null, null, isApplication);
+ }
+
+ private static boolean isApplicationEntity(String entityType) {
+ return TimelineEntityType.YARN_APPLICATION.toString().equals(entityType);
}
@Override
@@ -155,26 +174,46 @@ public class HBaseTimelineReaderImpl
}
NavigableSet<TimelineEntity> entities = new TreeSet<>();
- // Scan through part of the table to find the entities belong to one app and
- // one type
- Scan scan = new Scan();
- scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
- clusterId, userId, flowId, flowRunId, appId, entityType));
- scan.setMaxVersions(Integer.MAX_VALUE);
- ResultScanner scanner = entityTable.getResultScanner(hbaseConf, conn, scan);
- for (Result result : scanner) {
+ boolean isApplication = isApplicationEntity(entityType);
+ if (isApplication) {
+ // If getEntities() is called for an application, there can be at most
+ // one entity. If the entity passes the filter, it is returned. Otherwise,
+ // an empty set is returned.
+ byte[] rowKey = ApplicationRowKey.getRowKey(clusterId, userId, flowId,
+ flowRunId, appId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ Result result = applicationTable.getResult(hbaseConf, conn, get);
TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
- true, createdTimeBegin, createdTimeEnd,
- true, modifiedTimeBegin, modifiedTimeEnd,
- isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
- metricFilters);
- if (entity == null) {
- continue;
+ true, createdTimeBegin, createdTimeEnd, true, modifiedTimeBegin,
+ modifiedTimeEnd, isRelatedTo, relatesTo, infoFilters, configFilters,
+ eventFilters, metricFilters, isApplication);
+ if (entity != null) {
+ entities.add(entity);
}
- if (entities.size() > limit) {
- entities.pollLast();
+ } else {
+ // Scan through part of the table to find the entities belong to one app
+ // and one type
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
+ clusterId, userId, flowId, flowRunId, appId, entityType));
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ ResultScanner scanner =
+ entityTable.getResultScanner(hbaseConf, conn, scan);
+ for (Result result : scanner) {
+ TimelineEntity entity = parseEntity(result, fieldsToRetrieve,
+ true, createdTimeBegin, createdTimeEnd,
+ true, modifiedTimeBegin, modifiedTimeEnd,
+ isRelatedTo, relatesTo, infoFilters, configFilters, eventFilters,
+ metricFilters, isApplication);
+ if (entity == null) {
+ continue;
+ }
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ entities.add(entity);
}
- entities.add(entity);
}
return entities;
}
@@ -221,26 +260,37 @@ public class HBaseTimelineReaderImpl
boolean checkModifiedTime, long modifiedTimeBegin, long modifiedTimeEnd,
Map<String, Set<String>> isRelatedTo, Map<String, Set<String>> relatesTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> eventFilters, Set<String> metricFilters)
+ Set<String> eventFilters, Set<String> metricFilters,
+ boolean isApplication)
throws IOException {
if (result == null || result.isEmpty()) {
return null;
}
TimelineEntity entity = new TimelineEntity();
- entity.setType(EntityColumn.TYPE.readResult(result).toString());
- entity.setId(EntityColumn.ID.readResult(result).toString());
+ String entityType = isApplication ?
+ TimelineEntityType.YARN_APPLICATION.toString() :
+ EntityColumn.TYPE.readResult(result).toString();
+ entity.setType(entityType);
+ String entityId = isApplication ?
+ ApplicationColumn.ID.readResult(result).toString() :
+ EntityColumn.ID.readResult(result).toString();
+ entity.setId(entityId);
// fetch created time
- entity.setCreatedTime(
- ((Number) EntityColumn.CREATED_TIME.readResult(result)).longValue());
+ Number createdTime = isApplication ?
+ (Number)ApplicationColumn.CREATED_TIME.readResult(result) :
+ (Number)EntityColumn.CREATED_TIME.readResult(result);
+ entity.setCreatedTime(createdTime.longValue());
if (checkCreatedTime && (entity.getCreatedTime() < createdTimeBegin ||
entity.getCreatedTime() > createdTimeEnd)) {
return null;
}
// fetch modified time
- entity.setCreatedTime(
- ((Number) EntityColumn.MODIFIED_TIME.readResult(result)).longValue());
+ Number modifiedTime = isApplication ?
+ (Number)ApplicationColumn.MODIFIED_TIME.readResult(result) :
+ (Number)EntityColumn.MODIFIED_TIME.readResult(result);
+ entity.setModifiedTime(modifiedTime.longValue());
if (checkModifiedTime && (entity.getModifiedTime() < modifiedTimeBegin ||
entity.getModifiedTime() > modifiedTimeEnd)) {
return null;
@@ -250,7 +300,13 @@ public class HBaseTimelineReaderImpl
boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO);
+ if (isApplication) {
+ readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
+ true);
+ } else {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO,
+ true);
+ }
if (checkIsRelatedTo && !TimelineReaderUtils.matchRelations(
entity.getIsRelatedToEntities(), isRelatedTo)) {
return null;
@@ -265,7 +321,12 @@ public class HBaseTimelineReaderImpl
boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, EntityColumnPrefix.RELATES_TO);
+ if (isApplication) {
+ readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
+ false);
+ } else {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
+ }
if (checkRelatesTo && !TimelineReaderUtils.matchRelations(
entity.getRelatesToEntities(), relatesTo)) {
return null;
@@ -280,7 +341,11 @@ public class HBaseTimelineReaderImpl
boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.INFO);
+ if (isApplication) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
+ } else {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
+ }
if (checkInfo &&
!TimelineReaderUtils.matchFilters(entity.getInfo(), infoFilters)) {
return null;
@@ -295,7 +360,11 @@ public class HBaseTimelineReaderImpl
boolean checkConfigs = configFilters != null && configFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG);
+ if (isApplication) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
+ } else {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
+ }
if (checkConfigs && !TimelineReaderUtils.matchFilters(
entity.getConfigs(), configFilters)) {
return null;
@@ -310,7 +379,7 @@ public class HBaseTimelineReaderImpl
boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result);
+ readEvents(entity, result, isApplication);
if (checkEvents && !TimelineReaderUtils.matchEventFilters(
entity.getEvents(), eventFilters)) {
return null;
@@ -325,7 +394,7 @@ public class HBaseTimelineReaderImpl
boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
- readMetrics(entity, result);
+ readMetrics(entity, result, isApplication);
if (checkMetrics && !TimelineReaderUtils.matchMetricFilters(
entity.getMetrics(), metricFilters)) {
return null;
@@ -338,15 +407,15 @@ public class HBaseTimelineReaderImpl
return entity;
}
- private static void readRelationship(
- TimelineEntity entity, Result result, EntityColumnPrefix prefix)
- throws IOException {
+ private static <T> void readRelationship(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isRelatedTo) throws IOException {
// isRelatedTo and relatesTo are of type Map<String, Set<String>>
Map<String, Object> columns = prefix.readResults(result);
for (Map.Entry<String, Object> column : columns.entrySet()) {
for (String id : Separator.VALUES.splitEncoded(
column.getValue().toString())) {
- if (prefix.equals(EntityColumnPrefix.IS_RELATED_TO)) {
+ if (isRelatedTo) {
entity.addIsRelatedToEntity(column.getKey(), id);
} else {
entity.addRelatesToEntity(column.getKey(), id);
@@ -355,12 +424,12 @@ public class HBaseTimelineReaderImpl
}
}
- private static void readKeyValuePairs(
- TimelineEntity entity, Result result, EntityColumnPrefix prefix)
- throws IOException {
+ private static <T> void readKeyValuePairs(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isConfig) throws IOException {
// info and configuration are of type Map<String, Object or String>
Map<String, Object> columns = prefix.readResults(result);
- if (prefix.equals(EntityColumnPrefix.CONFIG)) {
+ if (isConfig) {
for (Map.Entry<String, Object> column : columns.entrySet()) {
entity.addConfig(column.getKey(), column.getKey().toString());
}
@@ -369,10 +438,11 @@ public class HBaseTimelineReaderImpl
}
}
- private static void readEvents(TimelineEntity entity, Result result)
- throws IOException {
+ private static void readEvents(TimelineEntity entity, Result result,
+ boolean isApplication) throws IOException {
Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<String, Object> eventsResult =
+ Map<String, Object> eventsResult = isApplication ?
+ ApplicationColumnPrefix.EVENT.readResults(result) :
EntityColumnPrefix.EVENT.readResults(result);
for (Map.Entry<String,Object> eventResult : eventsResult.entrySet()) {
Collection<String> tokens =
@@ -405,10 +475,16 @@ public class HBaseTimelineReaderImpl
entity.addEvents(eventsSet);
}
- private static void readMetrics(TimelineEntity entity, Result result)
- throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ private static void readMetrics(TimelineEntity entity, Result result,
+ boolean isApplication) throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult;
+ if (isApplication) {
+ metricsResult =
+ ApplicationColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ } else {
+ metricsResult =
+ EntityColumnPrefix.METRIC.readResultsWithTimestamps(result);
+ }
for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
metricsResult.entrySet()) {
TimelineMetric metric = new TimelineMetric();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 5290415..96192cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -38,9 +38,14 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
@@ -61,6 +66,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private Connection conn;
private TypedBufferedMutator<EntityTable> entityTable;
private TypedBufferedMutator<AppToFlowTable> appToFlowTable;
+ private TypedBufferedMutator<ApplicationTable> applicationTable;
private static final Log LOG = LogFactory
.getLog(HBaseTimelineWriterImpl.class);
@@ -84,6 +90,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
conn = ConnectionFactory.createConnection(hbaseConf);
entityTable = new EntityTable().getTableMutator(hbaseConf, conn);
appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
+ applicationTable = new ApplicationTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -102,18 +109,20 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
continue;
}
- byte[] rowKey =
+ // if the entity is the application, the destination is the application
+ // table
+ boolean isApplication = isApplicationEntity(te);
+ byte[] rowKey = isApplication ?
+ ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
+ appId) :
EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
te.getType(), te.getId());
- storeInfo(rowKey, te, flowVersion);
- storeEvents(rowKey, te.getEvents());
- storeConfig(rowKey, te.getConfigs());
- storeMetrics(rowKey, te.getMetrics());
- storeRelations(rowKey, te.getIsRelatedToEntities(),
- EntityColumnPrefix.IS_RELATED_TO);
- storeRelations(rowKey, te.getRelatesToEntities(),
- EntityColumnPrefix.RELATES_TO);
+ storeInfo(rowKey, te, flowVersion, isApplication);
+ storeEvents(rowKey, te.getEvents(), isApplication);
+ storeConfig(rowKey, te.getConfigs(), isApplication);
+ storeMetrics(rowKey, te.getMetrics(), isApplication);
+ storeRelations(rowKey, te, isApplication);
if (isApplicationCreated(te)) {
onApplicationCreated(
@@ -123,9 +132,12 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
return putStatus;
}
+ private static boolean isApplicationEntity(TimelineEntity te) {
+ return te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString());
+ }
+
private static boolean isApplicationCreated(TimelineEntity te) {
- if (te.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) {
- boolean isAppCreated = false;
+ if (isApplicationEntity(te)) {
for (TimelineEvent event : te.getEvents()) {
if (event.getId().equals(
ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
@@ -145,41 +157,74 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
rowKey, appToFlowTable, null, flowRunId);
}
+ private void storeRelations(byte[] rowKey, TimelineEntity te,
+ boolean isApplication) throws IOException {
+ if (isApplication) {
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ ApplicationColumnPrefix.IS_RELATED_TO, applicationTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ ApplicationColumnPrefix.RELATES_TO, applicationTable);
+ } else {
+ storeRelations(rowKey, te.getIsRelatedToEntities(),
+ EntityColumnPrefix.IS_RELATED_TO, entityTable);
+ storeRelations(rowKey, te.getRelatesToEntities(),
+ EntityColumnPrefix.RELATES_TO, entityTable);
+ }
+ }
+
/**
* Stores the Relations from the {@linkplain TimelineEntity} object
*/
- private void storeRelations(byte[] rowKey,
+ private <T> void storeRelations(byte[] rowKey,
Map<String, Set<String>> connectedEntities,
- EntityColumnPrefix entityColumnPrefix) throws IOException {
+ ColumnPrefix<T> columnPrefix, TypedBufferedMutator<T> table)
+ throws IOException {
for (Map.Entry<String, Set<String>> connectedEntity : connectedEntities
.entrySet()) {
// id3?id4?id5
String compoundValue =
Separator.VALUES.joinEncoded(connectedEntity.getValue());
- entityColumnPrefix.store(rowKey, entityTable, connectedEntity.getKey(),
- null, compoundValue);
+ columnPrefix.store(rowKey, table, connectedEntity.getKey(), null,
+ compoundValue);
}
}
/**
* Stores information from the {@linkplain TimelineEntity} object
*/
- private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion)
- throws IOException {
+ private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion,
+ boolean isApplication) throws IOException {
- EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
- EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
- EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
- te.getCreatedTime());
- EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
- te.getModifiedTime());
- EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
- Map<String, Object> info = te.getInfo();
- if (info != null) {
- for (Map.Entry<String, Object> entry : info.entrySet()) {
- EntityColumnPrefix.INFO.store(rowKey, entityTable, entry.getKey(),
- null, entry.getValue());
+ if (isApplication) {
+ ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId());
+ ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null,
+ te.getCreatedTime());
+ ApplicationColumn.MODIFIED_TIME.store(rowKey, applicationTable, null,
+ te.getModifiedTime());
+ ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null,
+ flowVersion);
+ Map<String, Object> info = te.getInfo();
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ ApplicationColumnPrefix.INFO.store(rowKey, applicationTable,
+ entry.getKey(), null, entry.getValue());
+ }
+ }
+ } else {
+ EntityColumn.ID.store(rowKey, entityTable, null, te.getId());
+ EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
+ EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
+ te.getCreatedTime());
+ EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
+ te.getModifiedTime());
+ EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
+ Map<String, Object> info = te.getInfo();
+ if (info != null) {
+ for (Map.Entry<String, Object> entry : info.entrySet()) {
+ EntityColumnPrefix.INFO.store(rowKey, entityTable, entry.getKey(),
+ null, entry.getValue());
+ }
}
}
}
@@ -187,14 +232,19 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
/**
* stores the config information from {@linkplain TimelineEntity}
*/
- private void storeConfig(byte[] rowKey, Map<String, String> config)
- throws IOException {
+ private void storeConfig(byte[] rowKey, Map<String, String> config,
+ boolean isApplication) throws IOException {
if (config == null) {
return;
}
for (Map.Entry<String, String> entry : config.entrySet()) {
- EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
+ if (isApplication) {
+ ApplicationColumnPrefix.CONFIG.store(rowKey, applicationTable,
+ entry.getKey(), null, entry.getValue());
+ } else {
+ EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
null, entry.getValue());
+ }
}
}
@@ -202,16 +252,21 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
* stores the {@linkplain TimelineMetric} information from the
* {@linkplain TimelineEvent} object
*/
- private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics)
- throws IOException {
+ private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
+ boolean isApplication) throws IOException {
if (metrics != null) {
for (TimelineMetric metric : metrics) {
String metricColumnQualifier = metric.getId();
Map<Long, Number> timeseries = metric.getValues();
for (Map.Entry<Long, Number> timeseriesEntry : timeseries.entrySet()) {
Long timestamp = timeseriesEntry.getKey();
- EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ if (isApplication) {
+ ApplicationColumnPrefix.METRIC.store(rowKey, applicationTable,
metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ } else {
+ EntityColumnPrefix.METRIC.store(rowKey, entityTable,
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ }
}
}
}
@@ -220,8 +275,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
/**
* Stores the events from the {@linkplain TimelineEvent} object
*/
- private void storeEvents(byte[] rowKey, Set<TimelineEvent> events)
- throws IOException {
+ private void storeEvents(byte[] rowKey, Set<TimelineEvent> events,
+ boolean isApplication) throws IOException {
if (events != null) {
for (TimelineEvent event : events) {
if (event != null) {
@@ -258,8 +313,13 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// convert back to string to avoid additional API on store.
String compoundColumnQualifier =
Bytes.toString(compoundColumnQualifierBytes);
- EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ if (isApplication) {
+ ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
compoundColumnQualifier, null, info.getValue());
+ } else {
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifier, null, info.getValue());
+ }
} // for info: eventInfo
}
}
@@ -279,6 +339,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
// flush all buffered mutators
entityTable.flush();
appToFlowTable.flush();
+ applicationTable.flush();
}
/**
@@ -288,15 +349,19 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
@Override
protected void serviceStop() throws Exception {
if (entityTable != null) {
- LOG.info("closing entity table");
+ LOG.info("closing the entity table");
// The close API performs flushing and releases any resources held
entityTable.close();
}
if (appToFlowTable != null) {
- LOG.info("closing app_flow table");
+ LOG.info("closing the app_flow table");
// The close API performs flushing and releases any resources held
appToFlowTable.close();
}
+ if (applicationTable != null) {
+ LOG.info("closing the application table");
+ applicationTable.close();
+ }
if (conn != null) {
LOG.info("closing the hbase Connection");
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 2c3897d..3a22ed6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
@@ -76,6 +77,12 @@ public class TimelineSchemaCreator {
if (StringUtils.isNotBlank(appToflowTableName)) {
hbaseConf.set(AppToFlowTable.TABLE_NAME_CONF_NAME, appToflowTableName);
}
+ // Grab the applicationTableName argument
+ String applicationTableName = commandLine.getOptionValue("a");
+ if (StringUtils.isNotBlank(applicationTableName)) {
+ hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
+ applicationTableName);
+ }
createAllTables(hbaseConf);
}
@@ -103,6 +110,8 @@ public class TimelineSchemaCreator {
o = new Option("a2f", "appToflowTableName", true, "app to flow table name");
o.setArgName("appToflowTableName");
+ o = new Option("a", "applicationTableName", true, "application table name");
+ o.setArgName("applicationTableName");
o.setRequired(false);
options.addOption(o);
@@ -132,6 +141,7 @@ public class TimelineSchemaCreator {
}
new EntityTable().createTable(admin, hbaseConf);
new AppToFlowTable().createTable(admin, hbaseConf);
+ new ApplicationTable().createTable(admin, hbaseConf);
} finally {
if (conn != null) {
conn.close();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
new file mode 100644
index 0000000..c028386
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -0,0 +1,136 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies fully qualified columns for the {@link ApplicationTable}.
+ */
+public enum ApplicationColumn implements Column<ApplicationTable> {
+
+ /**
+ * App id
+ */
+ ID(ApplicationColumnFamily.INFO, "id"),
+
+ /**
+ * When the application was created.
+ */
+ CREATED_TIME(ApplicationColumnFamily.INFO, "created_time"),
+
+ /**
+ * When it was modified.
+ */
+ MODIFIED_TIME(ApplicationColumnFamily.INFO, "modified_time"),
+
+ /**
+ * The version of the flow that this app belongs to.
+ */
+ FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version");
+
+ private final ColumnHelper<ApplicationTable> column;
+ private final ColumnFamily<ApplicationTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+
+ private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.column = new ColumnHelper<ApplicationTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<ApplicationTable> tableMutator, Long timestamp,
+ Object inputValue) throws IOException {
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumn} or null
+ */
+ public static final ApplicationColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (ApplicationColumn ac : ApplicationColumn.values()) {
+ // Find a match based only on name.
+ if (ac.getColumnQualifier().equals(columnQualifier)) {
+ return ac;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param name Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumn} or null if both
+ * arguments don't match.
+ */
+ public static final ApplicationColumn columnFor(
+ ApplicationColumnFamily columnFamily, String name) {
+
+ for (ApplicationColumn ac : ApplicationColumn.values()) {
+ // Find a match based column family and on name.
+ if (ac.columnFamily.equals(columnFamily)
+ && ac.getColumnQualifier().equals(name)) {
+ return ac;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
new file mode 100644
index 0000000..97e5f7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnFamily.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the application table column families.
+ */
+public enum ApplicationColumnFamily implements ColumnFamily<ApplicationTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons: a) the size
+ * of the config values can be very large and b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private ApplicationColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
new file mode 100644
index 0000000..cd9e845
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies partially qualified columns for the application table.
+ */
+public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(ApplicationColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(ApplicationColumnFamily.INFO, "r"),
+
+ /**
+ * To store TimelineEntity info values.
+ */
+ INFO(ApplicationColumnFamily.INFO, "i"),
+
+ /**
+ * Lifecycle events for an application
+ */
+ EVENT(ApplicationColumnFamily.INFO, "e"),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(ApplicationColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(ApplicationColumnFamily.METRICS, null);
+
+ private final ColumnHelper<ApplicationTable> column;
+ private final ColumnFamily<ApplicationTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
+ String columnPrefix) {
+ column = new ColumnHelper<ApplicationTable>(columnFamily);
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<ApplicationTable> tableMutator, String qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
+ */
+ public Object readResult(Result result, String qualifier) throws IOException {
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+ return column.readResult(result, columnQualifier);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public Map<String, Object> readResults(Result result) throws IOException {
+ return column.readResults(result, columnPrefixBytes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
+ */
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
+ readResultsWithTimestamps(Result result) throws IOException {
+ return column.readResultsWithTimestamps(result, columnPrefixBytes);
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumnPrefix} given a name, or null if there
+ * is no match. The following holds true: {@code columnFor(x) == columnFor(y)}
+ * if and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumnPrefix} or null
+ */
+ public static final ApplicationColumnPrefix columnFor(String columnPrefix) {
+
+ // Match column based on value, assume column family matches.
+ for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) {
+ // Find a match based only on name.
+ if (acp.getColumnPrefix().equals(columnPrefix)) {
+ return acp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link ApplicationColumnPrefix} given a name, or null if there
+ * is no match. The following holds true:
+ * {@code columnFor(a,x) == columnFor(b,y)} if and only if
+ * {@code (x == y == null)} or {@code a.equals(b) & x.equals(y)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link ApplicationColumnPrefix} or null if both
+ * arguments don't match.
+ */
+ public static final ApplicationColumnPrefix columnFor(
+ ApplicationColumnFamily columnFamily, String columnPrefix) {
+
+ // TODO: needs unit test to confirm and need to update javadoc to explain
+ // null prefix case.
+
+ for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) {
+ // Find a match based column family and on name.
+ if (acp.columnFamily.equals(columnFamily)
+ && (((columnPrefix == null) && (acp.getColumnPrefix() == null)) || (acp
+ .getColumnPrefix().equals(columnPrefix)))) {
+ return acp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
new file mode 100644
index 0000000..5f3868b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the application table.
+ */
+public class ApplicationRowKey {
+ // TODO: more methods are needed for this class.
+
+ // TODO: API needs to be cleaned up.
+
+ /**
+ * Constructs a row key for the application table as follows:
+ * {@code clusterId!userName!flowId!flowRunId!AppId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key
+ */
+ public static byte[] getRowKey(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, userId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(ApplicationRowKey.invert(flowRunId));
+ byte[] third = Bytes.toBytes(appId);
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Converts a timestamp into its inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invert(Long key) {
+ return Long.MAX_VALUE - key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
new file mode 100644
index 0000000..d2a2cb9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBaseSchemaConstants;
+
+/**
+ * The application table as column families info, config and metrics. Info
+ * stores information about a YARN application entity, config stores
+ * configuration data of a YARN application, metrics stores the metrics of a
+ * YARN application. This table is entirely analogous to the entity table but
+ * created for better performance.
+ *
+ * Example application table record:
+ *
+ * <pre>
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | clusterId! | id:appId | metricId1: | configKey1: |
+ * | userName! | | metricValue1 | configValue1 |
+ * | flowId! | created_time: | @timestamp1 | |
+ * | flowRunId! | 1392993084018 | | configKey2: |
+ * | AppId | | metriciD1: | configValue2 |
+ * | | modified_time: | metricValue2 | |
+ * | | 1392995081012 | @timestamp2 | |
+ * | | | | |
+ * | | i!infoKey: | metricId2: | |
+ * | | infoValue | metricValue1 | |
+ * | | | @timestamp2 | |
+ * | | r!relatesToKey: | | |
+ * | | id3?id4?id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey: | | |
+ * | | id7?id9?id6 | | |
+ * | | | | |
+ * | | e!eventId?timestamp?infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
+ * </pre>
+ */
+public class ApplicationTable extends BaseTable<ApplicationTable> {
+ /** application prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".application";
+
+ /** config param name that specifies the application table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * application table
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /** default value for application table name */
+ private static final String DEFAULT_TABLE_NAME =
+ "timelineservice.application";
+
+ /** default TTL is 30 days for metrics timeseries */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
+
+ private static final Log LOG = LogFactory.getLog(ApplicationTable.class);
+
+ public ApplicationTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor applicationTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(ApplicationColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ applicationTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(ApplicationColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ applicationTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(ApplicationColumnFamily.METRICS.getBytes());
+ applicationTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ applicationTableDescp
+ .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ applicationTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(applicationTableDescp,
+ TimelineHBaseSchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metrics in this table.
+ * @param hbaseConf configuration in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
new file mode 100644
index 0000000..c60e6f5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.application;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 58272ab..c8485c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -157,7 +157,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
- public <T> NavigableMap<String, NavigableMap<Long, T>>
+ public <V> NavigableMap<String, NavigableMap<Long, V>>
readResultsWithTimestamps(Result result) throws IOException {
return column.readResultsWithTimestamps(result, columnPrefixBytes);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51029a75/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index f657a14..9a8bd8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -40,36 +40,35 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* Example entity table record:
*
* <pre>
- * |--------------------------------------------------------------------|
- * | Row | Column Family | Column Family| Column Family|
- * | key | info | metrics | config |
- * |--------------------------------------------------------------------|
- * | userName! | id:entityId | metricId1: | configKey1: |
- * | clusterId! | | metricValue1 | configValue1 |
- * | flowId! | type:entityType | @timestamp1 | |
- * | flowRunId! | | | configKey2: |
- * | AppId! | created_time: | metriciD1: | configValue2 |
- * | entityType!| 1392993084018 | metricValue2 | |
- * | entityId | | @timestamp2 | |
- * | | modified_time: | | |
- * | | 1392995081012 | metricId2: | |
- * | | | metricValue1 | |
- * | | i!infoKey: | @timestamp2 | |
- * | | infoValue | | |
- * | | | | |
- * | | r!relatesToKey: | | |
- * | | id3?id4?id5 | | |
- * | | | | |
- * | | s!isRelatedToKey | | |
- * | | id7?id9?id6 | | |
- * | | | | |
- * | | e!eventId?eventInfoKey: | | |
- * | | eventInfoValue | | |
- * | | @timestamp | | |
- * | | | | |
- * | | flowVersion: | | |
- * | | versionValue | | |
- * |--------------------------------------------------------------------|
+ * |-------------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |-------------------------------------------------------------------------|
+ * | userName! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | | metricValue1 | configValue1 |
+ * | flowId! | type:entityType | @timestamp1 | |
+ * | flowRunId! | | | configKey2: |
+ * | AppId! | created_time: | metriciD1: | configValue2 |
+ * | entityType!| 1392993084018 | metricValue2 | |
+ * | entityId | | @timestamp2 | |
+ * | | modified_time: | | |
+ * | | 1392995081012 | metricId2: | |
+ * | | | metricValue1 | |
+ * | | i!infoKey: | @timestamp2 | |
+ * | | infoValue | | |
+ * | | | | |
+ * | | r!relatesToKey: | | |
+ * | | id3?id4?id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7?id9?id6 | | |
+ * | | | | |
+ * | | e!eventId?timestamp?infoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |-------------------------------------------------------------------------|
* </pre>
*/
public class EntityTable extends BaseTable<EntityTable> {
[42/50] [abbrv] hadoop git commit: YARN-3984. Adjusted the event
column key schema and avoided missing empty event. Contributed by Vrushali C.
Posted by sj...@apache.org.
YARN-3984. Adjusted the event column key schema and avoided missing empty event. Contributed by Vrushali C.
(cherry picked from commit 895ccfa1ab9e701f2908586e323249f670fe5544)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d1a1a22a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d1a1a22a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d1a1a22a
Branch: refs/heads/YARN-2928
Commit: d1a1a22a1341c06160a98de36a04e80c118df292
Parents: d93e636
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Aug 5 16:28:57 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../storage/HBaseTimelineWriterImpl.java | 23 +++-
.../storage/common/TimelineWriterUtils.java | 13 +++
.../storage/entity/EntityRowKey.java | 18 +---
.../storage/TestHBaseTimelineWriterImpl.java | 105 ++++++++++++++++---
5 files changed, 128 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1a1a22a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f5ce32a..a484040 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -121,6 +121,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3993. Changed to use the AM flag in ContainerContext determine AM
container in TestPerNodeTimelineCollectorsAuxService. (Sunil G via zjshen)
+ YARN-3984. Adjusted the event column key schema and avoided missing empty
+ event. (Vrushali C via zjshen)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1a1a22a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index cd2e76e..3173e87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -200,20 +201,32 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
"! Using the current timestamp");
eventTimestamp = System.currentTimeMillis();
}
+ byte[] columnQualifierFirst =
+ Bytes.toBytes(Separator.VALUES.encode(eventId));
+ byte[] columnQualifierWithTsBytes =
+ Separator.VALUES.join(columnQualifierFirst,
+ Bytes.toBytes(TimelineWriterUtils.invert(eventTimestamp)));
Map<String, Object> eventInfo = event.getInfo();
- if (eventInfo != null) {
+ if ((eventInfo == null) || (eventInfo.size() == 0)) {
+ // add separator since event key is empty
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(columnQualifierWithTsBytes,
+ null);
+ String compoundColumnQualifier =
+ Bytes.toString(compoundColumnQualifierBytes);
+ EntityColumnPrefix.EVENT.store(rowKey, entityTable,
+ compoundColumnQualifier, null, TimelineWriterUtils.EMPTY_BYTES);
+ } else {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
// eventId?infoKey
- byte[] columnQualifierFirst =
- Bytes.toBytes(Separator.VALUES.encode(eventId));
byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(columnQualifierFirst,
+ Separator.VALUES.join(columnQualifierWithTsBytes,
Bytes.toBytes(info.getKey()));
// convert back to string to avoid additional API on store.
String compoundColumnQualifier =
Bytes.toString(compoundColumnQualifierBytes);
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifier, eventTimestamp, info.getValue());
+ compoundColumnQualifier, null, info.getValue());
} // for info: eventInfo
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1a1a22a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
index 28a0b6a..c957bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -124,4 +124,17 @@ public class TimelineWriterUtils {
return segments;
}
+ /**
+ * Converts a timestamp into it's inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invert(Long key) {
+ return Long.MAX_VALUE - key;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1a1a22a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 61958c2..3e17ad0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
/**
* Represents a rowkey for the entity table.
@@ -47,7 +48,7 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId));
return Separator.QUALIFIERS.join(first, second, third);
}
@@ -70,24 +71,11 @@ public class EntityRowKey {
flowId));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
- byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] second = Bytes.toBytes(TimelineWriterUtils.invert(flowRunId));
byte[] third =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, te.getType(),
te.getId()));
return Separator.QUALIFIERS.join(first, second, third);
}
- /**
- * Converts a timestamp into it's inverse timestamp to be used in (row) keys
- * where we want to have the most recent timestamp in the top of the table
- * (scans start at the most recent timestamp first).
- *
- * @param key value to be inverted so that the latest version will be first in
- * a scan.
- * @return inverted long
- */
- public static long invert(Long key) {
- return Long.MAX_VALUE - key;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d1a1a22a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index 31cb5d2..fd5643d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -43,8 +43,8 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineWriterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
@@ -274,7 +274,7 @@ public class TestHBaseTimelineWriterImpl {
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(EntityRowKey.invert(runid), Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(TimelineWriterUtils.invert(runid), Bytes.toLong(rowKeyComponents[3]));
assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
@@ -317,7 +317,6 @@ public class TestHBaseTimelineWriterImpl {
byte[] startRow =
EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
s.setStartRow(startRow);
- s.setMaxVersions(Integer.MAX_VALUE);
Connection conn = ConnectionFactory.createConnection(c1);
ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
@@ -331,24 +330,23 @@ public class TestHBaseTimelineWriterImpl {
assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
entity));
- // check the events
- NavigableMap<String, NavigableMap<Long, Object>> eventsResult =
- EntityColumnPrefix.EVENT.readResultsWithTimestamps(result);
+ Map<String, Object> eventsResult =
+ EntityColumnPrefix.EVENT.readResults(result);
// there should be only one event
assertEquals(1, eventsResult.size());
// key name for the event
- String valueKey = eventId + Separator.VALUES.getValue() + expKey;
- for (Map.Entry<String, NavigableMap<Long, Object>> e :
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(Bytes.toBytes(eventId),
+ Bytes.toBytes(TimelineWriterUtils.invert(expTs)),
+ Bytes.toBytes(expKey));
+ String valueKey = Bytes.toString(compoundColumnQualifierBytes);
+ for (Map.Entry<String, Object> e :
eventsResult.entrySet()) {
// the value key must match
assertEquals(valueKey, e.getKey());
- NavigableMap<Long, Object> value = e.getValue();
+ Object value = e.getValue();
// there should be only one timestamp and value
- assertEquals(1, value.size());
- for (Map.Entry<Long, Object> e2: value.entrySet()) {
- assertEquals(expTs, e2.getKey());
- assertEquals(expVal, e2.getValue());
- }
+ assertEquals(expVal, value.toString());
}
}
}
@@ -360,6 +358,85 @@ public class TestHBaseTimelineWriterImpl {
}
}
+ @Test
+ public void testAdditionalEntityEmptyEventInfo() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ String eventId = "foo_event_id";
+ event.setId(eventId);
+ Long expTs = 1436512802000L;
+ event.setTimestamp(expTs);
+
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId("attempt_1329348432655_0001_m_000008_18");
+ entity.setType("FOO_ATTEMPT");
+ entity.addEvent(event);
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String cluster = "cluster_emptyeventkey";
+ String user = "user_emptyeventkey";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ s.setStartRow(startRow);
+ s.addFamily(EntityColumnFamily.INFO.getBytes());
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+
+ // check the row key
+ byte[] row1 = result.getRow();
+ assertTrue(isRowKeyCorrect(row1, cluster, user, flow, runid, appName,
+ entity));
+
+ Map<String, Object> eventsResult =
+ EntityColumnPrefix.EVENT.readResults(result);
+ // there should be only one event
+ assertEquals(1, eventsResult.size());
+ // key name for the event
+ byte[] compoundColumnQualifierWithTsBytes =
+ Separator.VALUES.join(Bytes.toBytes(eventId),
+ Bytes.toBytes(TimelineWriterUtils.invert(expTs)));
+ byte[] compoundColumnQualifierBytes =
+ Separator.VALUES.join(compoundColumnQualifierWithTsBytes,
+ null);
+ String valueKey = Bytes.toString(compoundColumnQualifierBytes);
+ for (Map.Entry<String, Object> e :
+ eventsResult.entrySet()) {
+ // the column qualifier key must match
+ assertEquals(valueKey, e.getKey());
+ Object value = e.getValue();
+ // value should be empty
+ assertEquals("", value.toString());
+ }
+ }
+ }
+ assertEquals(1, rowCount);
+
+ } finally {
+ hbi.stop();
+ hbi.close();
+ }
+ }
+
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
[12/50] [abbrv] hadoop git commit: YARN-3391. Clearly define flow ID/
flow run / flow version in API and storage. Contributed by Zhijie Shen
Posted by sj...@apache.org.
YARN-3391. Clearly define flow ID/ flow run / flow version in API and storage. Contributed by Zhijie Shen
(cherry picked from commit 68c6232f8423e55b4d152ef3d1d66aeb2d6a555e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84389771
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84389771
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84389771
Branch: refs/heads/YARN-2928
Commit: 84389771f99d3630e2c81bd6fcdc0f3e03fd2f93
Parents: 462c48a
Author: Junping Du <ju...@apache.org>
Authored: Thu Apr 9 18:04:27 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../applications/distributedshell/Client.java | 36 +++++++++++++------
.../distributedshell/TestDistributedShell.java | 13 +++----
.../yarn/util/timeline/TimelineUtils.java | 34 +++++++++++++++---
.../GetTimelineCollectorContextResponse.java | 17 +++++----
...tTimelineCollectorContextResponsePBImpl.java | 38 +++++++++++++-------
.../yarn_server_common_service_protos.proto | 5 +--
.../java/org/apache/hadoop/yarn/TestRPC.java | 7 ++--
.../collectormanager/NMCollectorService.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 18 ++++++----
.../application/Application.java | 6 ++--
.../application/ApplicationImpl.java | 27 +++++++++-----
.../application/TestApplication.java | 2 +-
.../yarn/server/nodemanager/webapp/MockApp.java | 23 +++++++++---
.../nodemanager/webapp/TestNMWebServices.java | 2 +-
.../server/resourcemanager/ClientRMService.java | 21 +++++++++++
.../resourcemanager/amlauncher/AMLauncher.java | 30 ++++++++--------
.../TestTimelineServiceClientIntegration.java | 2 +-
.../collector/AppLevelTimelineCollector.java | 10 +++---
.../collector/TimelineCollector.java | 4 +--
.../collector/TimelineCollectorContext.java | 32 +++++++++++------
.../collector/TimelineCollectorManager.java | 15 ++++----
.../storage/FileSystemTimelineWriterImpl.java | 13 +++----
.../timelineservice/storage/TimelineWriter.java | 7 ++--
...TestPerNodeTimelineCollectorsAuxService.java | 2 +-
.../collector/TestTimelineCollectorManager.java | 3 +-
.../TestFileSystemTimelineWriterImpl.java | 8 +++--
27 files changed, 256 insertions(+), 124 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a3a1f14..aea859a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -50,6 +50,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3334. NM uses timeline client to publish container metrics to new
timeline service. (Junping Du via zjshen)
+ YARN-3391. Clearly define flow ID/ flow run / flow version in API and storage.
+ (Zhijie Shen via junping_du)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index db69490..ff2f594 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -185,8 +185,9 @@ public class Client {
// Timeline domain writer access control
private String modifyACLs = null;
- private String flowId = null;
- private String flowRunId = null;
+ private String flowName = null;
+ private String flowVersion = null;
+ private long flowRunId = 0L;
// Command line options
private Options opts;
@@ -289,9 +290,11 @@ public class Client {
+ "modify the timeline entities in the given domain");
opts.addOption("create", false, "Flag to indicate whether to create the "
+ "domain specified with -domain.");
- opts.addOption("flow", true, "ID of the flow which the distributed shell "
+ opts.addOption("flow_name", true, "Flow name which the distributed shell "
+ "app belongs to");
- opts.addOption("flow_run", true, "ID of the flowrun which the distributed "
+ opts.addOption("flow_version", true, "Flow version which the distributed "
+ + "shell app belongs to");
+ opts.addOption("flow_run_id", true, "Flow run ID which the distributed "
+ "shell app belongs to");
opts.addOption("help", false, "Print usage");
opts.addOption("node_label_expression", true,
@@ -452,11 +455,19 @@ public class Client {
}
}
- if (cliParser.hasOption("flow")) {
- flowId = cliParser.getOptionValue("flow");
+ if (cliParser.hasOption("flow_name")) {
+ flowName = cliParser.getOptionValue("flow_name");
+ }
+ if (cliParser.hasOption("flow_version")) {
+ flowVersion = cliParser.getOptionValue("flow_version");
}
- if (cliParser.hasOption("flow_run")) {
- flowRunId = cliParser.getOptionValue("flow_run");
+ if (cliParser.hasOption("flow_run_id")) {
+ try {
+ flowRunId = Long.valueOf(cliParser.getOptionValue("flow_run_id"));
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(
+ "Flow run is not a valid long value", e);
+ }
}
return true;
}
@@ -550,10 +561,13 @@ public class Client {
}
Set<String> tags = new HashSet<String>();
- if (flowId != null) {
- tags.add(TimelineUtils.generateFlowIdTag(flowId));
+ if (flowName != null) {
+ tags.add(TimelineUtils.generateFlowNameTag(flowName));
+ }
+ if (flowVersion != null) {
+ tags.add(TimelineUtils.generateFlowVersionTag(flowVersion));
}
- if (flowRunId != null) {
+ if (flowRunId != 0) {
tags.add(TimelineUtils.generateFlowRunIdTag(flowRunId));
}
appContext.setApplicationTags(tags);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index daaad7c..cc5f5e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -237,9 +237,11 @@ public class TestDistributedShell {
args = mergeArgs(args, timelineArgs);
if (!defaultFlow) {
String[] flowArgs = {
- "--flow",
- "test_flow_id",
- "--flow_run",
+ "--flow_name",
+ "test_flow_name",
+ "--flow_version",
+ "test_flow_version",
+ "--flow_run_id",
"12345678"
};
args = mergeArgs(args, flowArgs);
@@ -368,7 +370,8 @@ public class TestDistributedShell {
UserGroupInformation.getCurrentUser().getShortUserName() +
(defaultFlow ? "/" +
TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
- "/0/" : "/test_flow_id/12345678/") + appId.toString();
+ "/1/1/" : "/test_flow_name/test_flow_version/12345678/") +
+ appId.toString();
// for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
String outputDirApp = basePath + "/DS_APP_ATTEMPT/";
@@ -393,8 +396,6 @@ public class TestDistributedShell {
String containerFileName = outputDirContainer + containerTimestampFileName;
File containerFile = new File(containerFileName);
Assert.assertTrue(containerFile.exists());
- String appTimeStamp = appId.getClusterTimestamp() + "_" + appId.getId()
- + "_";
// Verify NM posting container metrics info.
String outputDirContainerMetrics = basePath + "/" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index ce14562..137b7c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -44,7 +44,8 @@ import org.codehaus.jackson.map.ObjectMapper;
@Evolving
public class TimelineUtils {
- public static final String FLOW_ID_TAG_PREFIX = "TIMELINE_FLOW_ID_TAG";
+ public static final String FLOW_NAME_TAG_PREFIX = "TIMELINE_FLOW_NAME_TAG";
+ public static final String FLOW_VERSION_TAG_PREFIX = "TIMELINE_FLOW_VERSION_TAG";
public static final String FLOW_RUN_ID_TAG_PREFIX = "TIMELINE_FLOW_RUN_ID_TAG";
private static ObjectMapper mapper;
@@ -128,11 +129,36 @@ public class TimelineUtils {
return "flow_" + appId.getClusterTimestamp() + "_" + appId.getId();
}
- public static String generateFlowIdTag(String flowId) {
- return FLOW_ID_TAG_PREFIX + ":" + flowId;
+ /**
+ * Generate flow name tag
+ *
+ * @param flowName flow name that identifies a distinct flow application which
+ * can be run repeatedly over time
+ * @return
+ */
+ public static String generateFlowNameTag(String flowName) {
+ return FLOW_NAME_TAG_PREFIX + ":" + flowName;
+ }
+
+ /**
+ * Generate flow version tag
+ *
+ * @param flowVersion flow version that keeps track of the changes made to the
+ * flow
+ * @return
+ */
+ public static String generateFlowVersionTag(String flowVersion) {
+ return FLOW_VERSION_TAG_PREFIX + ":" + flowVersion;
}
- public static String generateFlowRunIdTag(String flowRunId) {
+ /**
+ * Generate flow run ID tag
+ *
+ * @param flowRunId flow run ID that identifies one instance (or specific
+ * execution) of that flow
+ * @return
+ */
+ public static String generateFlowRunIdTag(long flowRunId) {
return FLOW_RUN_ID_TAG_PREFIX + ":" + flowRunId;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
index 1558e2f..bd5c11e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/GetTimelineCollectorContextResponse.java
@@ -23,11 +23,12 @@ import org.apache.hadoop.yarn.util.Records;
public abstract class GetTimelineCollectorContextResponse {
public static GetTimelineCollectorContextResponse newInstance(
- String userId, String flowId, String flowRunId) {
+ String userId, String flowName, String flowVersion, long flowRunId) {
GetTimelineCollectorContextResponse response =
Records.newRecord(GetTimelineCollectorContextResponse.class);
response.setUserId(userId);
- response.setFlowId(flowId);
+ response.setFlowName(flowName);
+ response.setFlowVersion(flowVersion);
response.setFlowRunId(flowRunId);
return response;
}
@@ -36,11 +37,15 @@ public abstract class GetTimelineCollectorContextResponse {
public abstract void setUserId(String userId);
- public abstract String getFlowId();
+ public abstract String getFlowName();
- public abstract void setFlowId(String flowId);
+ public abstract void setFlowName(String flowName);
- public abstract String getFlowRunId();
+ public abstract String getFlowVersion();
- public abstract void setFlowRunId(String flowRunId);
+ public abstract void setFlowVersion(String flowVersion);
+
+ public abstract long getFlowRunId();
+
+ public abstract void setFlowRunId(long flowRunId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
index 6dc1f77..34713cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/GetTimelineCollectorContextResponsePBImpl.java
@@ -102,40 +102,52 @@ public class GetTimelineCollectorContextResponsePBImpl extends
}
@Override
- public String getFlowId() {
+ public String getFlowName() {
GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasFlowId()) {
+ if (!p.hasFlowName()) {
return null;
}
- return p.getFlowId();
+ return p.getFlowName();
}
@Override
- public void setFlowId(String flowId) {
+ public void setFlowName(String flowName) {
maybeInitBuilder();
- if (flowId == null) {
- builder.clearFlowId();
+ if (flowName == null) {
+ builder.clearFlowName();
return;
}
- builder.setFlowId(flowId);
+ builder.setFlowName(flowName);
}
@Override
- public String getFlowRunId() {
+ public String getFlowVersion() {
GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
- if (!p.hasFlowRunId()) {
+ if (!p.hasFlowVersion()) {
return null;
}
- return p.getFlowRunId();
+ return p.getFlowVersion();
}
@Override
- public void setFlowRunId(String flowRunId) {
+ public void setFlowVersion(String flowVersion) {
maybeInitBuilder();
- if (flowRunId == null) {
- builder.clearFlowRunId();
+ if (flowVersion == null) {
+ builder.clearFlowVersion();
return;
}
+ builder.setFlowVersion(flowVersion);
+ }
+
+ @Override
+ public long getFlowRunId() {
+ GetTimelineCollectorContextResponseProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getFlowRunId();
+ }
+
+ @Override
+ public void setFlowRunId(long flowRunId) {
+ maybeInitBuilder();
builder.setFlowRunId(flowRunId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index 2234752..22c4cf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -115,8 +115,9 @@ message GetTimelineCollectorContextRequestProto {
message GetTimelineCollectorContextResponseProto {
optional string user_id = 1;
- optional string flow_id = 2;
- optional string flow_run_id = 3;
+ optional string flow_name = 2;
+ optional string flow_version = 3;
+ optional int64 flow_run_id = 4;
}
message NMContainerStatusProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
index 3c9f57b..52ecd73 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestRPC.java
@@ -176,8 +176,9 @@ public class TestRPC {
GetTimelineCollectorContextResponse response =
proxy.getTimelineCollectorContext(request);
Assert.assertEquals("test_user_id", response.getUserId());
- Assert.assertEquals("test_flow_id", response.getFlowId());
- Assert.assertEquals("test_flow_run_id", response.getFlowRunId());
+ Assert.assertEquals("test_flow_name", response.getFlowName());
+ Assert.assertEquals("test_flow_version", response.getFlowVersion());
+ Assert.assertEquals(12345678L, response.getFlowRunId());
} catch (YarnException | IOException e) {
Assert.fail("RPC call failured is not expected here.");
}
@@ -374,7 +375,7 @@ public class TestRPC {
throws YarnException, IOException {
if (request.getApplicationId().getId() == 1) {
return GetTimelineCollectorContextResponse.newInstance(
- "test_user_id", "test_flow_id", "test_flow_run_id");
+ "test_user_id", "test_flow_name", "test_flow_version", 12345678L);
} else {
throw new YarnException("The application is not found.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index f37be23..dc5601f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -130,6 +130,6 @@ public class NMCollectorService extends CompositeService implements
" doesn't exist on NM.");
}
return GetTimelineCollectorContextResponse.newInstance(
- app.getUser(), app.getFlowId(), app.getFlowRunId());
+ app.getUser(), app.getFlowName(), app.getFlowVersion(), app.getFlowRunId());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8a60dcd..4dd9fa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -300,7 +300,7 @@ public class ContainerManagerImpl extends CompositeService implements
LOG.info("Recovering application " + appId);
//TODO: Recover flow and flow run ID
ApplicationImpl app = new ApplicationImpl(
- dispatcher, p.getUser(), null, null, appId, creds, context);
+ dispatcher, p.getUser(), null, null, 0, appId, creds, context);
context.getApplications().put(appId, app);
app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext));
}
@@ -870,12 +870,18 @@ public class ContainerManagerImpl extends CompositeService implements
try {
if (!serviceStopped) {
// Create the application
- String flowId = launchContext.getEnvironment().get(
- TimelineUtils.FLOW_ID_TAG_PREFIX);
- String flowRunId = launchContext.getEnvironment().get(
+ String flowName = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_NAME_TAG_PREFIX);
+ String flowVersion = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_VERSION_TAG_PREFIX);
+ String flowRunIdStr = launchContext.getEnvironment().get(
TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
- Application application = new ApplicationImpl(
- dispatcher, user, flowId, flowRunId, applicationID, credentials, context);
+ long flowRunId = 0L;
+ if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
+ flowRunId = Long.valueOf(flowRunIdStr);
+ }
+ Application application = new ApplicationImpl(dispatcher, user,
+ flowName, flowVersion, flowRunId, applicationID, credentials, context);
if (null == context.getApplications().putIfAbsent(applicationID,
application)) {
LOG.info("Creating a new application reference for app " + applicationID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
index 0c95193..5de3398 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
@@ -36,9 +36,11 @@ public interface Application extends EventHandler<ApplicationEvent> {
ApplicationState getApplicationState();
- String getFlowId();
+ String getFlowName();
- String getFlowRunId();
+ String getFlowVersion();
+
+ long getFlowRunId();
TimelineClient getTimelineClient();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index b0ab550..9319b6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -67,8 +67,9 @@ public class ApplicationImpl implements Application {
final Dispatcher dispatcher;
final String user;
- final String flowId;
- final String flowRunId;
+ final String flowName;
+ final String flowVersion;
+ final long flowRunId;
final ApplicationId appId;
final Credentials credentials;
Map<ApplicationAccessType, String> applicationACLs;
@@ -85,12 +86,13 @@ public class ApplicationImpl implements Application {
Map<ContainerId, Container> containers =
new HashMap<ContainerId, Container>();
- public ApplicationImpl(Dispatcher dispatcher, String user, String flowId,
- String flowRunId, ApplicationId appId, Credentials credentials,
- Context context) {
+ public ApplicationImpl(Dispatcher dispatcher, String user, String flowName,
+ String flowVersion, long flowRunId, ApplicationId appId,
+ Credentials credentials, Context context) {
this.dispatcher = dispatcher;
this.user = user;
- this.flowId = flowId;
+ this.flowName = flowName;
+ this.flowVersion = flowVersion;
this.flowRunId = flowRunId;
this.appId = appId;
this.credentials = credentials;
@@ -517,11 +519,18 @@ public class ApplicationImpl implements Application {
}
}
- public String getFlowId() {
- return flowId;
+ @Override
+ public String getFlowName() {
+ return flowName;
}
- public String getFlowRunId() {
+ @Override
+ public String getFlowVersion() {
+ return flowVersion;
+ }
+
+ @Override
+ public long getFlowRunId() {
return flowRunId;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 5303df5..3889d2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -531,7 +531,7 @@ public class TestApplication {
this.appId = BuilderUtils.newApplicationId(timestamp, id);
app = new ApplicationImpl(
- dispatcher, this.user, null, null, appId, null, context);
+ dispatcher, this.user, null, null, 0, appId, null, context);
containers = new ArrayList<Container>();
for (int i = 0; i < numContainers; i++) {
Container container = createMockedContainer(this.appId, i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
index 2ee572b..4d1be84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
@@ -40,8 +40,9 @@ public class MockApp implements Application {
Map<ContainerId, Container> containers = new HashMap<ContainerId, Container>();
ApplicationState appState;
Application app;
- String flowId;
- String flowRunId;
+ String flowName;
+ String flowVersion;
+ long flowRunId;
TimelineClient timelineClient = null;
public MockApp(int uniqId) {
@@ -59,6 +60,14 @@ public class MockApp implements Application {
appState = ApplicationState.NEW;
}
+ public MockApp(String user, long clusterTimeStamp, int uniqId,
+ String flowName, String flowVersion, long flowRunId) {
+ this(user, clusterTimeStamp, uniqId);
+ this.flowName = flowName;
+ this.flowVersion = flowVersion;
+ this.flowRunId = flowRunId;
+ }
+
public void setState(ApplicationState state) {
this.appState = state;
}
@@ -81,11 +90,15 @@ public class MockApp implements Application {
public void handle(ApplicationEvent event) {}
- public String getFlowId() {
- return flowId;
+ public String getFlowName() {
+ return flowName;
+ }
+
+ public String getFlowVersion() {
+ return flowVersion;
}
- public String getFlowRunId() {
+ public long getFlowRunId() {
return flowRunId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index 476f3df..ef5eb65 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -327,7 +327,7 @@ public class TestNMWebServices extends JerseyTestBase {
final String filename = "logfile1";
final String logMessage = "log message\n";
nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
- null, null, appId, null, nmContext));
+ null, null, 0, appId, null, nmContext));
MockContainer container = new MockContainer(appAttemptId,
new AsyncDispatcher(), new Configuration(), "user", appId, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 3e16165..666c8f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -154,6 +154,7 @@ import org.apache.hadoop.yarn.util.UTCClock;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.SettableFuture;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
/**
@@ -554,6 +555,26 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
+ // Sanity check for flow run
+ String value = null;
+ try {
+ for (String tag : submissionContext.getApplicationTags()) {
+ if (tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX + ":") ||
+ tag.startsWith(
+ TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) {
+ value = tag.substring(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1);
+ Long.valueOf(value);
+ }
+ }
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid to flow run: " + value +
+ ". Flow run should be a long integer", e);
+ RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
+ e.getMessage(), "ClientRMService",
+ "Exception in submitting application", applicationId);
+ throw RPCUtil.getRemoteException(e);
+ }
+
// Check whether app has already been put into rmContext,
// If it is, simply return the response
if (rmContext.getRMApps().get(applicationId) != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 5c0b02b..37aa3e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -222,22 +222,9 @@ public class AMLauncher implements Runnable {
// Set flow context info
for (String tag :
rmContext.getRMApps().get(applicationId).getApplicationTags()) {
- if (tag.startsWith(TimelineUtils.FLOW_ID_TAG_PREFIX + ":") ||
- tag.startsWith(TimelineUtils.FLOW_ID_TAG_PREFIX.toLowerCase() + ":")) {
- String value = tag.substring(
- TimelineUtils.FLOW_ID_TAG_PREFIX.length() + 1);
- if (!value.isEmpty()) {
- environment.put(TimelineUtils.FLOW_ID_TAG_PREFIX, value);
- }
- }
- if (tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX + ":") ||
- tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) {
- String value = tag.substring(
- TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1);
- if (!value.isEmpty()) {
- environment.put(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, value);
- }
- }
+ setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, tag);
+ setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, tag);
+ setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, tag);
}
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
@@ -259,6 +246,17 @@ public class AMLauncher implements Runnable {
container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}
+ private static void setFlowTags(
+ Map<String, String> environment, String tagPrefix, String tag) {
+ if (tag.startsWith(tagPrefix + ":") ||
+ tag.startsWith(tagPrefix.toLowerCase() + ":")) {
+ String value = tag.substring(tagPrefix.length() + 1);
+ if (!value.isEmpty()) {
+ environment.put(tagPrefix, value);
+ }
+ }
+ }
+
@VisibleForTesting
protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
Token<AMRMTokenIdentifier> amrmToken =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index c8b9625..54c806c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -97,7 +97,7 @@ public class TestTimelineServiceClientIntegration {
mock(CollectorNodemanagerProtocol.class);
try {
GetTimelineCollectorContextResponse response =
- GetTimelineCollectorContextResponse.newInstance(null, null, null);
+ GetTimelineCollectorContextResponse.newInstance(null, null, null, 0L);
when(protocol.getTimelineCollectorContext(any(
GetTimelineCollectorContextRequest.class))).thenReturn(response);
} catch (YarnException | IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index 60ddde5..5bc70e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -54,10 +54,12 @@ public class AppLevelTimelineCollector extends TimelineCollector {
// context info from NM.
// Current user usually is not the app user, but keep this field non-null
context.setUserId(UserGroupInformation.getCurrentUser().getShortUserName());
- // Use app ID to generate a default flow ID for orphan app
- context.setFlowId(TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId));
- // Set the flow run ID to 0 if it's an orphan app
- context.setFlowRunId("0");
+ // Use app ID to generate a default flow name for orphan app
+ context.setFlowName(TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId));
+ // Set the flow version to string 1 if it's an orphan app
+ context.setFlowVersion("1");
+ // Set the flow run ID to 1 if it's an orphan app
+ context.setFlowRunId(1L);
context.setAppId(appId.toString());
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 677feb1..f1d3d72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -100,8 +100,8 @@ public abstract class TimelineCollector extends CompositeService {
TimelineCollectorContext context = getTimelineEntityContext();
return writer.write(context.getClusterId(), context.getUserId(),
- context.getFlowId(), context.getFlowRunId(), context.getAppId(),
- entities);
+ context.getFlowName(), context.getFlowVersion(), context.getFlowRunId(),
+ context.getAppId(), entities);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
index c1a10a6..6cc477f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
@@ -22,19 +22,21 @@ public class TimelineCollectorContext {
private String clusterId;
private String userId;
- private String flowId;
- private String flowRunId;
+ private String flowName;
+ private String flowVersion;
+ private long flowRunId;
private String appId;
public TimelineCollectorContext() {
- this(null, null, null, null, null);
+ this(null, null, null, null, 0L, null);
}
public TimelineCollectorContext(String clusterId, String userId,
- String flowId, String flowRunId, String appId) {
+ String flowName, String flowVersion, long flowRunId, String appId) {
this.clusterId = clusterId;
this.userId = userId;
- this.flowId = flowId;
+ this.flowName = flowName;
+ this.flowVersion = flowVersion;
this.flowRunId = flowRunId;
this.appId = appId;
}
@@ -55,19 +57,27 @@ public class TimelineCollectorContext {
this.userId = userId;
}
- public String getFlowId() {
- return flowId;
+ public String getFlowName() {
+ return flowName;
}
- public void setFlowId(String flowId) {
- this.flowId = flowId;
+ public void setFlowName(String flowName) {
+ this.flowName = flowName;
}
- public String getFlowRunId() {
+ public String getFlowVersion() {
+ return flowVersion;
+ }
+
+ public void setFlowVersion(String flowVersion) {
+ this.flowVersion = flowVersion;
+ }
+
+ public long getFlowRunId() {
return flowRunId;
}
- public void setFlowRunId(String flowRunId) {
+ public void setFlowRunId(long flowRunId) {
this.flowRunId = flowRunId;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 5f23c25..9a566a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.http.lib.StaticUserWebFilter;
-import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -273,12 +272,16 @@ public class TimelineCollectorManager extends CompositeService {
if (userId != null && !userId.isEmpty()) {
collector.getTimelineEntityContext().setUserId(userId);
}
- String flowId = response.getFlowId();
- if (flowId != null && !flowId.isEmpty()) {
- collector.getTimelineEntityContext().setFlowId(flowId);
+ String flowName = response.getFlowName();
+ if (flowName != null && !flowName.isEmpty()) {
+ collector.getTimelineEntityContext().setFlowName(flowName);
}
- String flowRunId = response.getFlowRunId();
- if (flowRunId != null && !flowRunId.isEmpty()) {
+ String flowVersion = response.getFlowVersion();
+ if (flowVersion != null && !flowVersion.isEmpty()) {
+ collector.getTimelineEntityContext().setFlowVersion(flowVersion);
+ }
+ long flowRunId = response.getFlowRunId();
+ if (flowRunId != 0L) {
collector.getTimelineEntityContext().setFlowRunId(flowRunId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index 41b6ac9..dd8ad06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -65,22 +65,23 @@ public class FileSystemTimelineWriterImpl extends AbstractService
@Override
public TimelineWriteResponse write(String clusterId, String userId,
- String flowId, String flowRunId, String appId,
+ String flowName, String flowVersion, long flowRunId, String appId,
TimelineEntities entities) throws IOException {
TimelineWriteResponse response = new TimelineWriteResponse();
for (TimelineEntity entity : entities.getEntities()) {
- write(clusterId, userId, flowId, flowRunId, appId, entity, response);
+ write(clusterId, userId, flowName, flowVersion, flowRunId, appId, entity,
+ response);
}
return response;
}
- private void write(String clusterId, String userId,
- String flowId, String flowRunId, String appId, TimelineEntity entity,
+ private void write(String clusterId, String userId, String flowName,
+ String flowVersion, long flowRun, String appId, TimelineEntity entity,
TimelineWriteResponse response) throws IOException {
PrintWriter out = null;
try {
- String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowId,
- flowRunId, appId, entity.getType());
+ String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowName,
+ flowVersion, String.valueOf(flowRun), appId, entity.getType());
String fileName = dir + entity.getId() + TIMELINE_SERVICE_STORAGE_EXTENSION;
out = new PrintWriter(new BufferedWriter(new FileWriter(fileName, true)));
out.println(TimelineUtils.dumpTimelineRecordtoJSON(entity));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 492e3a9..467bcec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -41,8 +41,9 @@ public interface TimelineWriter extends Service {
*
* @param clusterId context cluster ID
* @param userId context user ID
- * @param flowId context flow ID
- * @param flowRunId context flow run ID
+ * @param flowName context flow name
+ * @param flowVersion context flow version
+ * @param flowRunId
* @param appId context app ID
* @param data
* a {@link TimelineEntities} object.
@@ -50,7 +51,7 @@ public interface TimelineWriter extends Service {
* @throws IOException
*/
TimelineWriteResponse write(String clusterId, String userId,
- String flowId, String flowRunId, String appId,
+ String flowName, String flowVersion, long flowRunId, String appId,
TimelineEntities data) throws IOException;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index 1de8d6d..abbe13a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -162,7 +162,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
GetTimelineCollectorContextResponse response =
- GetTimelineCollectorContextResponse.newInstance(null, null, null);
+ GetTimelineCollectorContextResponse.newInstance(null, null, null, 0L);
try {
when(nmCollectorService.getTimelineCollectorContext(any(
GetTimelineCollectorContextRequest.class))).thenReturn(response);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
index 36bda85..c662998 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -146,7 +147,7 @@ public class TestTimelineCollectorManager {
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
GetTimelineCollectorContextResponse response =
- GetTimelineCollectorContextResponse.newInstance(null, null, null);
+ GetTimelineCollectorContextResponse.newInstance(null, null, null, 0L);
try {
when(nmCollectorService.getTimelineCollectorContext(any(
GetTimelineCollectorContextRequest.class))).thenReturn(response);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84389771/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index 407b5f6..50a9f60 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -57,11 +57,13 @@ public class TestFileSystemTimelineWriterImpl {
fsi = new FileSystemTimelineWriterImpl();
fsi.init(new YarnConfiguration());
fsi.start();
- fsi.write("cluster_id", "user_id", "flow_id", "flow_run_id", "app_id", te);
+ fsi.write("cluster_id", "user_id", "flow_name", "flow_version", 12345678L,
+ "app_id", te);
String fileName = fsi.getOutputRoot() +
- "/entities/cluster_id/user_id/flow_id/flow_run_id/app_id/" + type +
- "/" + id + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ "/entities/cluster_id/user_id/flow_name/flow_version/12345678/app_id/" +
+ type + "/" + id +
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
Path path = Paths.get(fileName);
File f = new File(fileName);
assertTrue(f.exists() && !f.isDirectory());
[20/50] [abbrv] hadoop git commit: YARN-3134. Implemented Phoenix
timeline writer to access HBase backend. Contributed by Li Lu.
Posted by sj...@apache.org.
YARN-3134. Implemented Phoenix timeline writer to access HBase backend. Contributed by Li Lu.
(cherry picked from commit b3b791be466be79e4e964ad068f7a6ec701e22e1)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9b794ab1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9b794ab1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9b794ab1
Branch: refs/heads/YARN-2928
Commit: 9b794ab16dde75a088469f6393aab73f198e2a9a
Parents: ed95a79
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 8 19:08:02 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:11 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 11 +
.../hadoop-yarn-server-timelineservice/pom.xml | 17 +
.../collector/TimelineCollector.java | 13 +-
.../collector/TimelineCollectorManager.java | 19 +
.../storage/PhoenixTimelineWriterImpl.java | 509 +++++++++++++++++++
.../storage/TestPhoenixTimelineWriterImpl.java | 125 +++++
.../storage/TestTimelineWriterImpl.java | 74 +++
8 files changed, 760 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 82f95d6..a290c22 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -64,6 +64,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3562. unit tests failures and issues found from findbug from earlier
ATS checkins (Naganarasimha G R via sjlee)
+ YARN-3134. Implemented Phoenix timeline writer to access HBase backend. (Li
+ Lu via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 114851f..d25d1d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -485,6 +485,17 @@
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Ignore SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING warnings for Timeline Phoenix storage. -->
+ <!-- Since we're using dynamic columns, we have to generate SQL statements dynamically -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.timelineservice.storage.PhoenixTimelineWriterImpl" />
+ <Or>
+ <Method name="storeEntityVariableLengthFields" />
+ <Method name="storeEvents" />
+ <Method name="storeMetrics" />
+ <Method name="write" />
+ </Or>
+ </Match>
<!-- Following fields are used in ErrorsAndWarningsBlock, which is not a part of analysis of findbugs -->
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index f974aee..f62230f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -120,6 +120,23 @@
<artifactId>mockito-all</artifactId>
<scope>test</scope>
</dependency>
+
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <version>4.3.0</version>
+ <exclusions>
+ <!-- Exclude jline from here -->
+ <exclusion>
+ <artifactId>jline</artifactId>
+ <groupId>jline</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 4eced5b..bb7db12 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -27,11 +27,8 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
/**
@@ -55,11 +52,6 @@ public abstract class TimelineCollector extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- writer = ReflectionUtils.newInstance(conf.getClass(
- YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
- FileSystemTimelineWriterImpl.class,
- TimelineWriter.class), conf);
- writer.init(conf);
}
@Override
@@ -70,11 +62,10 @@ public abstract class TimelineCollector extends CompositeService {
@Override
protected void serviceStop() throws Exception {
super.serviceStop();
- writer.stop();
}
- public TimelineWriter getWriter() {
- return writer;
+ protected void setWriter(TimelineWriter w) {
+ this.writer = w;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 7b3da6b..953d9b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -23,9 +23,14 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
import java.util.Collections;
import java.util.HashMap;
@@ -42,6 +47,19 @@ public abstract class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
+ protected TimelineWriter writer;
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ writer = ReflectionUtils.newInstance(conf.getClass(
+ YarnConfiguration.TIMELINE_SERVICE_WRITER_CLASS,
+ FileSystemTimelineWriterImpl.class,
+ TimelineWriter.class), conf);
+ writer.init(conf);
+ super.serviceInit(conf);
+ }
+
+
// access to this map is synchronized with the map itself
private final Map<ApplicationId, TimelineCollector> collectors =
Collections.synchronizedMap(
@@ -69,6 +87,7 @@ public abstract class TimelineCollectorManager extends AbstractService {
// initialize, start, and add it to the collection so it can be
// cleaned up when the parent shuts down
collector.init(getConfig());
+ collector.setWriter(writer);
collector.start();
collectors.put(appId, collector);
LOG.info("the collector for " + appId + " was added");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
new file mode 100644
index 0000000..af8a233
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
@@ -0,0 +1,509 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+
+@Private
+@Unstable
+public class PhoenixTimelineWriterImpl extends AbstractService
+ implements TimelineWriter {
+
+ private static final Log LOG
+ = LogFactory.getLog(PhoenixTimelineWriterImpl.class);
+ private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
+ = "timeline_cf_placeholder";
+ // These lists are not taking effects in table creations.
+ private static final String[] PHOENIX_STORAGE_PK_LIST
+ = {"cluster", "user", "flow_name", "flow_version", "flow_run", "app_id",
+ "type", "entity_id"};
+ private static final String[] TIMELINE_EVENT_EXTRA_PK_LIST =
+ {"timestamp", "event_id"};
+ private static final String[] TIMELINE_METRIC_EXTRA_PK_LIST =
+ {"metric_id"};
+ /** Default Phoenix JDBC driver name */
+ private static final String DRIVER_CLASS_NAME
+ = "org.apache.phoenix.jdbc.PhoenixDriver";
+
+ /** Default Phoenix timeline entity table name */
+ @VisibleForTesting
+ static final String ENTITY_TABLE_NAME = "timeline_entity";
+ /** Default Phoenix event table name */
+ @VisibleForTesting
+ static final String EVENT_TABLE_NAME = "timeline_event";
+ /** Default Phoenix metric table name */
+ @VisibleForTesting
+ static final String METRIC_TABLE_NAME = "metric_singledata";
+
+ /** Default Phoenix timeline config column family */
+ private static final String CONFIG_COLUMN_FAMILY = "c.";
+ /** Default Phoenix timeline info column family */
+ private static final String INFO_COLUMN_FAMILY = "i.";
+ /** Default Phoenix event info column family */
+ private static final String EVENT_INFO_COLUMN_FAMILY = "ei.";
+ /** Default Phoenix isRelatedTo column family */
+ private static final String IS_RELATED_TO_FAMILY = "ir.";
+ /** Default Phoenix relatesTo column family */
+ private static final String RELATES_TO_FAMILY = "rt.";
+ /** Default separator for Phoenix storage */
+ private static final String PHOENIX_STORAGE_SEPARATOR = ";";
+
+ /** Connection string to the deployed Phoenix cluster */
+ static final String CONN_STRING = "jdbc:phoenix:localhost:2181:/hbase";
+
+ PhoenixTimelineWriterImpl() {
+ super((PhoenixTimelineWriterImpl.class.getName()));
+ }
+
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ createTables();
+ super.init(conf);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ super.serviceStop();
+ }
+
+ @Override
+ public TimelineWriteResponse write(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRunId, String appId,
+ TimelineEntities entities) throws IOException {
+ TimelineWriteResponse response = new TimelineWriteResponse();
+ TimelineCollectorContext currContext = new TimelineCollectorContext(
+ clusterId, userId, flowName, flowVersion, flowRunId, appId);
+ String sql = "UPSERT INTO " + ENTITY_TABLE_NAME
+ + " (" + StringUtils.join(PHOENIX_STORAGE_PK_LIST, ",")
+ + ", creation_time, modified_time, configs) "
+ + "VALUES (" + StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length)
+ + "?, ?, ?)";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TimelineEntity write SQL: " + sql);
+ }
+
+ try (Connection conn = getConnection();
+ PreparedStatement ps = conn.prepareStatement(sql)) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ int idx = setStringsForPrimaryKey(ps, currContext, entity, 1);
+ ps.setLong(idx++, entity.getCreatedTime());
+ ps.setLong(idx++, entity.getModifiedTime());
+ String configKeys = StringUtils.join(
+ entity.getConfigs().keySet(), PHOENIX_STORAGE_SEPARATOR);
+ ps.setString(idx++, configKeys);
+ ps.execute();
+
+ storeEntityVariableLengthFields(entity, currContext, conn);
+ storeEvents(entity, currContext, conn);
+ storeMetrics(entity, currContext, conn);
+
+ conn.commit();
+ }
+ } catch (SQLException se) {
+ LOG.error("Failed to add entity to Phoenix " + se.getMessage());
+ throw new IOException(se);
+ } catch (Exception e) {
+ LOG.error("Exception on getting connection: " + e.getMessage());
+ throw new IOException(e);
+ }
+ return response;
+ }
+
+ /**
+ * Aggregates the entity information to the timeline store based on which
+ * track this entity is to be rolled up to The tracks along which aggregations
+ * are to be done are given by {@link TimelineAggregationTrack}
+ *
+ * Any errors occurring for individual write request objects will be reported
+ * in the response.
+ *
+ * @param data
+ * a {@link TimelineEntity} object
+ * a {@link TimelineAggregationTrack} enum value
+ * @return a {@link TimelineWriteResponse} object.
+ * @throws IOException
+ */
+ @Override
+ public TimelineWriteResponse aggregate(TimelineEntity data,
+ TimelineAggregationTrack track) throws IOException {
+ return null;
+
+ }
+
+ // Utility functions
+ @Private
+ @VisibleForTesting
+ static Connection getConnection() throws IOException {
+ Connection conn;
+ try {
+ Class.forName(DRIVER_CLASS_NAME);
+ conn = DriverManager.getConnection(CONN_STRING);
+ conn.setAutoCommit(false);
+ } catch (SQLException se) {
+ LOG.error("Failed to connect to phoenix server! "
+ + se.getLocalizedMessage());
+ throw new IOException(se);
+ } catch (ClassNotFoundException e) {
+ LOG.error("Class not found! " + e.getLocalizedMessage());
+ throw new IOException(e);
+ }
+ return conn;
+ }
+
+ private void createTables() throws Exception {
+ // Create tables if necessary
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ // Table schema defined as in YARN-3134.
+ String sql = "CREATE TABLE IF NOT EXISTS " + ENTITY_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
+ + "flow_run UNSIGNED_LONG NOT NULL, "
+ + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
+ + "entity_id VARCHAR NOT NULL, "
+ + "creation_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + "configs VARCHAR, "
+ + CONFIG_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
+ + INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + IS_RELATED_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
+ + RELATES_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
+ + "type, entity_id))";
+ stmt.executeUpdate(sql);
+ sql = "CREATE TABLE IF NOT EXISTS " + EVENT_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
+ + "flow_run UNSIGNED_LONG NOT NULL, "
+ + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
+ + "entity_id VARCHAR NOT NULL, "
+ + "timestamp UNSIGNED_LONG NOT NULL, event_id VARCHAR NOT NULL, "
+ + EVENT_INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
+ + "type, entity_id, timestamp DESC, event_id))";
+ stmt.executeUpdate(sql);
+ sql = "CREATE TABLE IF NOT EXISTS " + METRIC_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
+ + "flow_run UNSIGNED_LONG NOT NULL, "
+ + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
+ + "entity_id VARCHAR NOT NULL, "
+ + "metric_id VARCHAR NOT NULL, "
+ + "singledata VARBINARY, "
+ + "time UNSIGNED_LONG "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
+ + "type, entity_id, metric_id))";
+ stmt.executeUpdate(sql);
+ conn.commit();
+ } catch (SQLException se) {
+ LOG.error("Failed in init data " + se.getLocalizedMessage());
+ throw se;
+ }
+ return;
+ }
+
+ private static class DynamicColumns<K> {
+ static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
+ static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
+ String columnFamilyPrefix;
+ String type;
+ Set<K> columns;
+
+ public DynamicColumns(String columnFamilyPrefix, String type,
+ Set<K> keyValues) {
+ this.columnFamilyPrefix = columnFamilyPrefix;
+ this.columns = keyValues;
+ this.type = type;
+ }
+ }
+
+ private static <K> StringBuilder appendColumnsSQL(
+ StringBuilder colNames, DynamicColumns<K> cfInfo) {
+ // Prepare the sql template by iterating through all keys
+ for (K key : cfInfo.columns) {
+ colNames.append(",").append(cfInfo.columnFamilyPrefix)
+ .append(key.toString()).append(cfInfo.type);
+ }
+ return colNames;
+ }
+
+ private static <K, V> int setValuesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos,
+ boolean converToBytes) throws SQLException {
+ int idx = startPos;
+ for (Map.Entry<K, V> entry : keyValues.entrySet()) {
+ V value = entry.getValue();
+ if (value instanceof Collection) {
+ ps.setString(idx++, StringUtils.join(
+ (Collection) value, PHOENIX_STORAGE_SEPARATOR));
+ } else {
+ if (converToBytes) {
+ try {
+ ps.setBytes(idx++, GenericObjectMapper.write(entry.getValue()));
+ } catch (IOException ie) {
+ LOG.error("Exception in converting values into bytes "
+ + ie.getMessage());
+ throw new SQLException(ie);
+ }
+ } else {
+ ps.setString(idx++, value.toString());
+ }
+ }
+ }
+ return idx;
+ }
+
+ private static <K, V> int setBytesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, true);
+ }
+
+ private static <K, V> int setStringsForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, false);
+ }
+
+ private static int setStringsForPrimaryKey(PreparedStatement ps,
+ TimelineCollectorContext context, TimelineEntity entity, int startPos)
+ throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getClusterId());
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++,
+ context.getFlowName());
+ ps.setString(idx++, context.getFlowVersion());
+ ps.setLong(idx++, context.getFlowRunId());
+ ps.setString(idx++, context.getAppId());
+ ps.setString(idx++, entity.getType());
+ ps.setString(idx++, entity.getId());
+ return idx;
+ }
+
+ private static void storeEntityVariableLengthFields(TimelineEntity entity,
+ TimelineCollectorContext context, Connection conn) throws SQLException {
+ int numPlaceholders = 0;
+ StringBuilder columnDefs = new StringBuilder(
+ StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
+ if (entity.getConfigs() != null) {
+ Set<String> keySet = entity.getConfigs().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ CONFIG_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (entity.getInfo() != null) {
+ Set<String> keySet = entity.getInfo().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (entity.getIsRelatedToEntities() != null) {
+ Set<String> keySet = entity.getIsRelatedToEntities().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ IS_RELATED_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (entity.getRelatesToEntities() != null) {
+ Set<String> keySet = entity.getRelatesToEntities().keySet();
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ RELATES_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
+ keySet));
+ numPlaceholders += keySet.size();
+ }
+ if (numPlaceholders == 0) {
+ return;
+ }
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length));
+ // numPlaceholders >= 1 now
+ placeholders.append("?")
+ .append(StringUtils.repeat(",?", numPlaceholders - 1));
+ String sqlVariableLengthFields = new StringBuilder("UPSERT INTO ")
+ .append(ENTITY_TABLE_NAME).append(" (").append(columnDefs)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for variable length fields: "
+ + sqlVariableLengthFields);
+ }
+ // Use try with resource statement for the prepared statement
+ try (PreparedStatement psVariableLengthFields =
+ conn.prepareStatement(sqlVariableLengthFields)) {
+ int idx = setStringsForPrimaryKey(
+ psVariableLengthFields, context, entity, 1);
+ if (entity.getConfigs() != null) {
+ idx = setStringsForColumnFamily(
+ psVariableLengthFields, entity.getConfigs(), idx);
+ }
+ if (entity.getInfo() != null) {
+ idx = setBytesForColumnFamily(
+ psVariableLengthFields, entity.getInfo(), idx);
+ }
+ if (entity.getIsRelatedToEntities() != null) {
+ idx = setStringsForColumnFamily(
+ psVariableLengthFields, entity.getIsRelatedToEntities(), idx);
+ }
+ if (entity.getRelatesToEntities() != null) {
+ idx = setStringsForColumnFamily(
+ psVariableLengthFields, entity.getRelatesToEntities(), idx);
+ }
+ psVariableLengthFields.execute();
+ }
+ }
+
+ private static void storeMetrics(TimelineEntity entity,
+ TimelineCollectorContext context, Connection conn) throws SQLException {
+ if (entity.getMetrics() == null) {
+ return;
+ }
+ Set<TimelineMetric> metrics = entity.getMetrics();
+ for (TimelineMetric metric : metrics) {
+ StringBuilder sqlColumns = new StringBuilder(
+ StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
+ sqlColumns.append(",")
+ .append(StringUtils.join(TIMELINE_METRIC_EXTRA_PK_LIST, ","));
+ sqlColumns.append(",").append("singledata, time");
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
+ .append(StringUtils.repeat("?,", TIMELINE_METRIC_EXTRA_PK_LIST.length));
+ placeholders.append("?, ?");
+ String sqlMetric = new StringBuilder("UPSERT INTO ")
+ .append(METRIC_TABLE_NAME).append(" (").append(sqlColumns)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for metric: " + sqlMetric);
+ }
+ try (PreparedStatement psMetrics = conn.prepareStatement(sqlMetric)) {
+ if (metric.getType().equals(TimelineMetric.Type.TIME_SERIES)) {
+ LOG.warn("The incoming timeline metric contains time series data, "
+ + "which is currently not supported by Phoenix storage. "
+ + "Time series will be truncated. ");
+ }
+ int idx = setStringsForPrimaryKey(psMetrics, context, entity, 1);
+ psMetrics.setString(idx++, metric.getId());
+ Iterator<Map.Entry<Long, Number>> currNumIter =
+ metric.getValues().entrySet().iterator();
+ if (currNumIter.hasNext()) {
+ // TODO: support time series storage
+ Map.Entry<Long, Number> currEntry = currNumIter.next();
+ psMetrics.setBytes(idx++,
+ GenericObjectMapper.write(currEntry.getValue()));
+ psMetrics.setLong(idx++, currEntry.getKey());
+ } else {
+ psMetrics.setBytes(idx++, GenericObjectMapper.write(null));
+ LOG.warn("The incoming metric contains an empty value set. ");
+ }
+ psMetrics.execute();
+ } catch (IOException ie) {
+ LOG.error("Exception on converting single data to bytes: "
+ + ie.getMessage());
+ throw new SQLException(ie);
+ }
+ }
+ }
+
+ private static void storeEvents(TimelineEntity entity,
+ TimelineCollectorContext context, Connection conn) throws SQLException {
+ if (entity.getEvents() == null) {
+ return;
+ }
+ Set<TimelineEvent> events = entity.getEvents();
+ for (TimelineEvent event : events) {
+ // We need this number to check if the incoming event's info field is empty
+ int numPlaceholders = 0;
+ StringBuilder sqlColumns = new StringBuilder(
+ StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
+ sqlColumns.append(",")
+ .append(StringUtils.join(TIMELINE_EVENT_EXTRA_PK_LIST, ","));
+ appendColumnsSQL(sqlColumns, new DynamicColumns<>(
+ EVENT_INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
+ event.getInfo().keySet()));
+ numPlaceholders += event.getInfo().keySet().size();
+ if (numPlaceholders == 0) {
+ continue;
+ }
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
+ .append(StringUtils.repeat("?,", TIMELINE_EVENT_EXTRA_PK_LIST.length));
+ // numPlaceholders >= 1 now
+ placeholders.append("?")
+ .append(StringUtils.repeat(",?", numPlaceholders - 1));
+ String sqlEvents = new StringBuilder("UPSERT INTO ")
+ .append(EVENT_TABLE_NAME).append(" (").append(sqlColumns)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for events: " + sqlEvents);
+ }
+ try (PreparedStatement psEvent = conn.prepareStatement(sqlEvents)) {
+ int idx = setStringsForPrimaryKey(psEvent, context, entity, 1);
+ psEvent.setLong(idx++, event.getTimestamp());
+ psEvent.setString(idx++, event.getId());
+ setBytesForColumnFamily(psEvent, event.getInfo(), idx);
+ psEvent.execute();
+ }
+ }
+ }
+
+ // WARNING: This method will permanently drop a table!
+ @Private
+ @VisibleForTesting
+ void dropTable(String tableName) throws Exception {
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ String sql = "DROP TABLE " + tableName;
+ stmt.executeUpdate(sql);
+ } catch (SQLException se) {
+ LOG.error("Failed in dropping entity table " + se.getLocalizedMessage());
+ throw se;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
new file mode 100644
index 0000000..a55893e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
@@ -0,0 +1,125 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
+public class TestPhoenixTimelineWriterImpl {
+ private PhoenixTimelineWriterImpl writer;
+
+ @Before
+ public void setup() throws Exception {
+ // TODO: launch a miniphoenix cluster, or else we're directly operating on
+ // the active Phoenix cluster
+ YarnConfiguration conf = new YarnConfiguration();
+ writer = createPhoenixWriter(conf);
+ }
+
+ @Ignore
+ @Test
+ public void testPhoenixWriterBasic() throws Exception {
+ // Set up a list of timeline entities and write them back to Phoenix
+ int numEntity = 12;
+ TimelineEntities te =
+ TestTimelineWriterImpl.getStandardTestTimelineEntities(numEntity);
+ writer.write("cluster_1", "user1", "testFlow", "version1", 1l, "app_test_1", te);
+ // Verify if we're storing all entities
+ String sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME;
+ verifySQLWithCount(sql, numEntity, "Number of entities should be ");
+ // Check config (half of all entities)
+ sql = "SELECT COUNT(c.config) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(c.config VARCHAR) ";
+ verifySQLWithCount(sql, (numEntity / 2),
+ "Number of entities with config should be ");
+ // Check info (half of all entities)
+ sql = "SELECT COUNT(i.info1) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(i.info1 VARBINARY) ";
+ verifySQLWithCount(sql, (numEntity / 2),
+ "Number of entities with info should be ");
+ // Check config and info (a quarter of all entities)
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
+ + "(c.config VARCHAR, i.info1 VARBINARY) "
+ + "WHERE c.config IS NOT NULL AND i.info1 IS NOT NULL";
+ verifySQLWithCount(sql, (numEntity / 4),
+ "Number of entities with both config and info should be ");
+ // Check relatesToEntities and isRelatedToEntities
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
+ + "(rt.testType VARCHAR, ir.testType VARCHAR) "
+ + "WHERE rt.testType IS NOT NULL AND ir.testType IS NOT NULL";
+ verifySQLWithCount(sql, numEntity - 2,
+ "Number of entities with both relatesTo and isRelatedTo should be ");
+ // Check event
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.EVENT_TABLE_NAME;
+ verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
+ // Check metrics
+ sql = "SELECT COUNT(entity_id) FROM "
+ + PhoenixTimelineWriterImpl.METRIC_TABLE_NAME;
+ verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
+ }
+
+ @After
+ public void cleanup() throws Exception {
+ // Note: it is assumed that we're working on a test only cluster, or else
+ // this cleanup process will drop the entity table.
+ writer.dropTable(PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME);
+ writer.dropTable(PhoenixTimelineWriterImpl.EVENT_TABLE_NAME);
+ writer.dropTable(PhoenixTimelineWriterImpl.METRIC_TABLE_NAME);
+ writer.serviceStop();
+ }
+
+ private static PhoenixTimelineWriterImpl createPhoenixWriter(
+ YarnConfiguration conf) throws Exception{
+ PhoenixTimelineWriterImpl myWriter = new PhoenixTimelineWriterImpl();
+ myWriter.serviceInit(conf);
+ return myWriter;
+ }
+
+ private void verifySQLWithCount(String sql, int targetCount, String message)
+ throws Exception{
+ try (
+ Statement stmt =
+ PhoenixTimelineWriterImpl.getConnection().createStatement();
+ ResultSet rs = stmt.executeQuery(sql)) {
+ assertTrue("Result set empty on statement " + sql, rs.next());
+ assertNotNull("Fail to execute query " + sql, rs);
+ assertEquals(message + " " + targetCount, targetCount, rs.getInt(1));
+ } catch (SQLException se) {
+ fail("SQL exception on query: " + sql
+ + " With exception message: " + se.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9b794ab1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
new file mode 100644
index 0000000..7a7afc0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+public class TestTimelineWriterImpl {
+ static TimelineEntities getStandardTestTimelineEntities(int listSize) {
+ TimelineEntities te = new TimelineEntities();
+ for (int i = 0; i < listSize; i++) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello" + i;
+ String type = "testType";
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(1425016501000L + i);
+ entity.setModifiedTime(1425016502000L + i);
+ if (i > 0) {
+ entity.addRelatesToEntity(type, "hello" + i);
+ entity.addRelatesToEntity(type, "hello" + (i - 1));
+ }
+ if (i < listSize - 1) {
+ entity.addIsRelatedToEntity(type, "hello" + i);
+ entity.addIsRelatedToEntity(type, "hello" + (i + 1));
+ }
+ int category = i % 4;
+ switch (category) {
+ case 0:
+ entity.addConfig("config", "config" + i);
+ // Fall through deliberately
+ case 1:
+ entity.addInfo("info1", new Integer(i));
+ entity.addInfo("info2", "helloworld");
+ // Fall through deliberately
+ case 2:
+ break;
+ case 3:
+ entity.addConfig("config", "config" + i);
+ TimelineEvent event = new TimelineEvent();
+ event.setId("test event");
+ event.setTimestamp(1425016501100L + i);
+ event.addInfo("test_info", "content for " + entity.getId());
+ event.addInfo("test_info1", new Integer(i));
+ entity.addEvent(event);
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("HDFS_BYTES_READ");
+ metric.addValue(1425016501100L + i, 8000 + i);
+ entity.addMetric(metric);
+ break;
+ }
+ te.addEntity(entity);
+ }
+ return te;
+ }
+}
[11/50] [abbrv] hadoop git commit: YARN-3390. Reuse
TimelineCollectorManager for RM (Zhijie Shen via sjlee)
Posted by sj...@apache.org.
YARN-3390. Reuse TimelineCollectorManager for RM (Zhijie Shen via sjlee)
(cherry picked from commit 58221188811e0f61d842dac89e1f4ad4fd8aa182)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9e8d853
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9e8d853
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9e8d853
Branch: refs/heads/YARN-2928
Commit: e9e8d853ba904c89270e901a4a5327f85bd06050
Parents: 8438977
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Apr 24 16:56:23 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:09 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../resourcemanager/RMActiveServiceContext.java | 13 +-
.../server/resourcemanager/RMAppManager.java | 15 +-
.../yarn/server/resourcemanager/RMContext.java | 7 +-
.../server/resourcemanager/RMContextImpl.java | 12 +-
.../server/resourcemanager/ResourceManager.java | 14 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 15 ++
.../timelineservice/RMTimelineCollector.java | 111 --------
.../RMTimelineCollectorManager.java | 75 ++++++
.../TestTimelineServiceClientIntegration.java | 12 +-
.../collector/AppLevelTimelineCollector.java | 2 +-
.../collector/NodeTimelineCollectorManager.java | 223 ++++++++++++++++
.../PerNodeTimelineCollectorsAuxService.java | 15 +-
.../collector/TimelineCollector.java | 2 +-
.../collector/TimelineCollectorManager.java | 259 +++----------------
.../collector/TimelineCollectorWebService.java | 23 +-
.../TestNMTimelineCollectorManager.java | 160 ++++++++++++
...TestPerNodeTimelineCollectorsAuxService.java | 24 +-
.../collector/TestTimelineCollectorManager.java | 160 ------------
19 files changed, 585 insertions(+), 559 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index aea859a..43d292a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -53,6 +53,8 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3391. Clearly define flow ID/ flow run / flow version in API and storage.
(Zhijie Shen via junping_du)
+ YARN-3390. Reuse TimelineCollectorManager for RM (Zhijie Shen via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index d79e542..17346ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
@@ -95,7 +95,7 @@ public class RMActiveServiceContext {
private ApplicationMasterService applicationMasterService;
private RMApplicationHistoryWriter rmApplicationHistoryWriter;
private SystemMetricsPublisher systemMetricsPublisher;
- private RMTimelineCollector timelineCollector;
+ private RMTimelineCollectorManager timelineCollectorManager;
private RMNodeLabelsManager nodeLabelManager;
private long epoch;
@@ -377,14 +377,15 @@ public class RMActiveServiceContext {
@Private
@Unstable
- public RMTimelineCollector getRMTimelineCollector() {
- return timelineCollector;
+ public RMTimelineCollectorManager getRMTimelineCollectorManager() {
+ return timelineCollectorManager;
}
@Private
@Unstable
- public void setRMTimelineCollector(RMTimelineCollector timelineCollector) {
- this.timelineCollector = timelineCollector;
+ public void setRMTimelineCollectorManager(
+ RMTimelineCollectorManager timelineCollectorManager) {
+ this.timelineCollectorManager = timelineCollectorManager;
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 6fd1838..759e020 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -338,12 +338,13 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
submissionContext.setPriority(appPriority);
// Create RMApp
- RMAppImpl application = new RMAppImpl(applicationId, rmContext, this.conf,
- submissionContext.getApplicationName(), user,
- submissionContext.getQueue(), submissionContext, this.scheduler,
- this.masterService, submitTime, submissionContext.getApplicationType(),
- submissionContext.getApplicationTags(), amReq);
-
+ RMAppImpl application =
+ new RMAppImpl(applicationId, rmContext, this.conf,
+ submissionContext.getApplicationName(), user,
+ submissionContext.getQueue(),
+ submissionContext, this.scheduler, this.masterService,
+ submitTime, submissionContext.getApplicationType(),
+ submissionContext.getApplicationTags(), amReq);
// Concurrent app submissions with same applicationId will fail here
// Concurrent app submissions with different applicationIds will not
// influence each other
@@ -354,6 +355,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
LOG.warn(message);
throw new YarnException(message);
}
+ // Start timeline collector for the submitted app
+ application.startTimelineCollector();
// Inform the ACLs Manager
this.applicationACLsManager.addApplication(applicationId,
submissionContext.getAMContainerSpec().getApplicationACLs());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 05fee99..5ef799a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
/**
* Context of the ResourceManager.
@@ -110,9 +110,10 @@ public interface RMContext {
SystemMetricsPublisher getSystemMetricsPublisher();
- void setRMTimelineCollector(RMTimelineCollector timelineCollector);
+ void setRMTimelineCollectorManager(
+ RMTimelineCollectorManager timelineCollectorManager);
- RMTimelineCollector getRMTimelineCollector();
+ RMTimelineCollectorManager getRMTimelineCollectorManager();
ConfigurationProvider getConfigurationProvider();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 8543cec..8038972 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRen
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMDelegationTokenSecretManager;
-import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.util.Clock;
import com.google.common.annotations.VisibleForTesting;
@@ -355,14 +355,14 @@ public class RMContextImpl implements RMContext {
}
@Override
- public void setRMTimelineCollector(
- RMTimelineCollector timelineCollector) {
- activeServiceContext.setRMTimelineCollector(timelineCollector);
+ public void setRMTimelineCollectorManager(
+ RMTimelineCollectorManager timelineCollectorManager) {
+ activeServiceContext.setRMTimelineCollectorManager(timelineCollectorManager);
}
@Override
- public RMTimelineCollector getRMTimelineCollector() {
- return activeServiceContext.getRMTimelineCollector();
+ public RMTimelineCollectorManager getRMTimelineCollectorManager() {
+ return activeServiceContext.getRMTimelineCollectorManager();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index e5c72cd..525c88b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -96,11 +96,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEv
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.security.DelegationTokenRenewer;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollector;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebApp;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilter;
import org.apache.hadoop.yarn.server.security.http.RMAuthenticationFilterInitializer;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.webproxy.AppReportFetcher;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxy;
@@ -368,8 +368,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
return new RMApplicationHistoryWriter();
}
- private RMTimelineCollector createRMTimelineCollector() {
- return new RMTimelineCollector();
+ private RMTimelineCollectorManager createRMTimelineCollectorManager() {
+ return new RMTimelineCollectorManager(rmContext);
}
protected SystemMetricsPublisher createSystemMetricsPublisher() {
@@ -493,10 +493,10 @@ public class ResourceManager extends CompositeService implements Recoverable {
addService(systemMetricsPublisher);
rmContext.setSystemMetricsPublisher(systemMetricsPublisher);
- RMTimelineCollector timelineCollector =
- createRMTimelineCollector();
- addService(timelineCollector);
- rmContext.setRMTimelineCollector(timelineCollector);
+ RMTimelineCollectorManager timelineCollectorManager =
+ createRMTimelineCollectorManager();
+ addService(timelineCollectorManager);
+ rmContext.setRMTimelineCollectorManager(timelineCollectorManager);
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index c4f5270..bbbbb93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -91,6 +91,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
@@ -475,6 +477,17 @@ public class RMAppImpl implements RMApp, Recoverable {
YarnConfiguration.DEFAULT_RM_MAX_LOG_AGGREGATION_DIAGNOSTICS_IN_MEMORY);
}
+ public void startTimelineCollector() {
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(applicationId);
+ rmContext.getRMTimelineCollectorManager().putIfAbsent(
+ applicationId, collector);
+ }
+
+ public void stopTimelineCollector() {
+ rmContext.getRMTimelineCollectorManager().remove(applicationId);
+ }
+
@Override
public ApplicationId getApplicationId() {
return this.applicationId;
@@ -1288,6 +1301,8 @@ public class RMAppImpl implements RMApp, Recoverable {
.applicationFinished(app, finalState);
app.rmContext.getSystemMetricsPublisher()
.appFinished(app, finalState, app.finishTime);
+
+ app.stopTimelineCollector();
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
deleted file mode 100644
index 4ea7a03..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollector.java
+++ /dev/null
@@ -1,111 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.timelineservice;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsEventType;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-
-/**
- * This class is responsible for posting application and appattempt lifecycle
- * related events to timeline service V2
- */
-@Private
-@Unstable
-public class RMTimelineCollector extends TimelineCollector {
- private static final Log LOG = LogFactory.getLog(RMTimelineCollector.class);
-
- public RMTimelineCollector() {
- super("Resource Manager TimelineCollector");
- }
-
- private Dispatcher dispatcher;
-
- private boolean publishSystemMetricsForV2;
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- publishSystemMetricsForV2 =
- conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
- && conf.getBoolean(
- YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
- YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
-
- if (publishSystemMetricsForV2) {
- // having separate dispatcher to avoid load on RMDispatcher
- LOG.info("RMTimelineCollector has been configured to publish"
- + " System Metrics in ATS V2");
- dispatcher = new AsyncDispatcher();
- dispatcher.register(SystemMetricsEventType.class,
- new ForwardingEventHandler());
- } else {
- LOG.warn("RMTimelineCollector has not been configured to publish"
- + " System Metrics in ATS V2");
- }
- super.serviceInit(conf);
- }
-
- @Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- }
-
- protected void handleSystemMetricsEvent(SystemMetricsEvent event) {
- switch (event.getType()) {
- default:
- LOG.error("Unknown SystemMetricsEvent type: " + event.getType());
- }
- }
-
- @Override
- protected TimelineCollectorContext getTimelineEntityContext() {
- // TODO address in YARN-3390.
- return null;
- }
-
- /**
- * EventHandler implementation which forward events to SystemMetricsPublisher.
- * Making use of it, SystemMetricsPublisher can avoid to have a public handle
- * method.
- */
- private final class ForwardingEventHandler implements
- EventHandler<SystemMetricsEvent> {
-
- @Override
- public void handle(SystemMetricsEvent event) {
- handleSystemMetricsEvent(event);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
new file mode 100644
index 0000000..25e0e0f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class RMTimelineCollectorManager extends TimelineCollectorManager {
+ private RMContext rmContext;
+
+ public RMTimelineCollectorManager(RMContext rmContext) {
+ super(RMTimelineCollectorManager.class.getName());
+ this.rmContext = rmContext;
+ }
+
+ @Override
+ public void postPut(ApplicationId appId, TimelineCollector collector) {
+ RMApp app = rmContext.getRMApps().get(appId);
+ if (app == null) {
+ throw new YarnRuntimeException(
+ "Unable to get the timeline collector context info for a non-existing app " +
+ appId);
+ }
+ String userId = app.getUser();
+ if (userId != null && !userId.isEmpty()) {
+ collector.getTimelineEntityContext().setUserId(userId);
+ }
+ for (String tag : app.getApplicationTags()) {
+ String[] parts = tag.split(":", 2);
+ if (parts.length != 2 || parts[1].isEmpty()) {
+ continue;
+ }
+ switch (parts[0]) {
+ case TimelineUtils.FLOW_NAME_TAG_PREFIX:
+ collector.getTimelineEntityContext().setFlowName(parts[1]);
+ break;
+ case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
+ collector.getTimelineEntityContext().setFlowVersion(parts[1]);
+ break;
+ case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
+ collector.getTimelineEntityContext().setFlowRunId(
+ Long.valueOf(parts[1]));
+ break;
+ default:
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 54c806c..0bdb68a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+import org.apache.hadoop.yarn.server.timelineservice.collector.NodeTimelineCollectorManager;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -42,13 +42,13 @@ import org.junit.Test;
import java.io.IOException;
public class TestTimelineServiceClientIntegration {
- private static TimelineCollectorManager collectorManager;
+ private static NodeTimelineCollectorManager collectorManager;
private static PerNodeTimelineCollectorsAuxService auxService;
@BeforeClass
public static void setupClass() throws Exception {
try {
- collectorManager = new MyTimelineCollectorManager();
+ collectorManager = new MockNodeTimelineCollectorManager();
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
collectorManager);
@@ -85,9 +85,9 @@ public class TestTimelineServiceClientIntegration {
}
}
- private static class MyTimelineCollectorManager extends
- TimelineCollectorManager {
- public MyTimelineCollectorManager() {
+ private static class MockNodeTimelineCollectorManager extends
+ NodeTimelineCollectorManager {
+ public MockNodeTimelineCollectorManager() {
super();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index 5bc70e3..fa32211 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -75,7 +75,7 @@ public class AppLevelTimelineCollector extends TimelineCollector {
}
@Override
- protected TimelineCollectorContext getTimelineEntityContext() {
+ public TimelineCollectorContext getTimelineEntityContext() {
return context;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
new file mode 100644
index 0000000..03ac770
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -0,0 +1,223 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.util.HashMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.http.HttpServer2;
+import org.apache.hadoop.http.lib.StaticUserWebFilter;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
+import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
+
+import com.google.common.annotations.VisibleForTesting;
+
+
+/**
+ *
+ * It is a singleton, and instances should be obtained via
+ * {@link #getInstance()}.
+ *
+ */
+@Private
+@Unstable
+public class NodeTimelineCollectorManager extends TimelineCollectorManager {
+ private static final Log LOG =
+ LogFactory.getLog(NodeTimelineCollectorManager.class);
+ private static final NodeTimelineCollectorManager INSTANCE =
+ new NodeTimelineCollectorManager();
+
+
+ // REST server for this collector manager
+ private HttpServer2 timelineRestServer;
+
+ private String timelineRestServerBindAddress;
+
+ private CollectorNodemanagerProtocol nmCollectorService;
+
+ private InetSocketAddress nmCollectorServiceAddress;
+
+ static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
+
+ static NodeTimelineCollectorManager getInstance() {
+ return INSTANCE;
+ }
+
+ @VisibleForTesting
+ protected NodeTimelineCollectorManager() {
+ super(NodeTimelineCollectorManager.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ this.nmCollectorServiceAddress = conf.getSocketAddr(
+ YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
+ super.serviceInit(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ nmCollectorService = getNMCollectorService();
+ startWebApp();
+ super.serviceStart();
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ if (timelineRestServer != null) {
+ timelineRestServer.stop();
+ }
+ super.serviceStop();
+ }
+
+ @Override
+ public void postPut(ApplicationId appId, TimelineCollector collector) {
+ try {
+ // Get context info from NM
+ updateTimelineCollectorContext(appId, collector);
+ // Report to NM if a new collector is added.
+ reportNewCollectorToNM(appId);
+ } catch (YarnException | IOException e) {
+ // throw exception here as it cannot be used if failed communicate with NM
+ LOG.error("Failed to communicate with NM Collector Service for " + appId);
+ throw new YarnRuntimeException(e);
+ }
+ }
+
+ /**
+ * Launch the REST web server for this collector manager
+ */
+ private void startWebApp() {
+ Configuration conf = getConfig();
+ String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
+ try {
+ Configuration confForInfoServer = new Configuration(conf);
+ confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
+ HttpServer2.Builder builder = new HttpServer2.Builder()
+ .setName("timeline")
+ .setConf(conf)
+ .addEndpoint(URI.create(
+ (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
+ bindAddress));
+ timelineRestServer = builder.build();
+ // TODO: replace this by an authentication filter in future.
+ HashMap<String, String> options = new HashMap<>();
+ String username = conf.get(HADOOP_HTTP_STATIC_USER,
+ DEFAULT_HADOOP_HTTP_STATIC_USER);
+ options.put(HADOOP_HTTP_STATIC_USER, username);
+ HttpServer2.defineFilter(timelineRestServer.getWebAppContext(),
+ "static_user_filter_timeline",
+ StaticUserWebFilter.StaticUserFilter.class.getName(),
+ options, new String[] {"/*"});
+
+ timelineRestServer.addJerseyResourcePackage(
+ TimelineCollectorWebService.class.getPackage().getName() + ";"
+ + GenericExceptionHandler.class.getPackage().getName() + ";"
+ + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
+ "/*");
+ timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY, this);
+ timelineRestServer.start();
+ } catch (Exception e) {
+ String msg = "The per-node collector webapp failed to start.";
+ LOG.error(msg, e);
+ throw new YarnRuntimeException(msg, e);
+ }
+ //TODO: We need to think of the case of multiple interfaces
+ this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
+ timelineRestServer.getConnectorAddress(0));
+ LOG.info("Instantiated the per-node collector webapp at " +
+ timelineRestServerBindAddress);
+ }
+
+ private void reportNewCollectorToNM(ApplicationId appId)
+ throws YarnException, IOException {
+ ReportNewCollectorInfoRequest request =
+ ReportNewCollectorInfoRequest.newInstance(appId,
+ this.timelineRestServerBindAddress);
+ LOG.info("Report a new collector for application: " + appId +
+ " to the NM Collector Service.");
+ nmCollectorService.reportNewCollectorInfo(request);
+ }
+
+ private void updateTimelineCollectorContext(
+ ApplicationId appId, TimelineCollector collector)
+ throws YarnException, IOException {
+ GetTimelineCollectorContextRequest request =
+ GetTimelineCollectorContextRequest.newInstance(appId);
+ LOG.info("Get timeline collector context for " + appId);
+ GetTimelineCollectorContextResponse response =
+ nmCollectorService.getTimelineCollectorContext(request);
+ String userId = response.getUserId();
+ if (userId != null && !userId.isEmpty()) {
+ collector.getTimelineEntityContext().setUserId(userId);
+ }
+ String flowName = response.getFlowName();
+ if (flowName != null && !flowName.isEmpty()) {
+ collector.getTimelineEntityContext().setFlowName(flowName);
+ }
+ String flowVersion = response.getFlowVersion();
+ if (flowVersion != null && !flowVersion.isEmpty()) {
+ collector.getTimelineEntityContext().setFlowVersion(flowVersion);
+ }
+ long flowRunId = response.getFlowRunId();
+ if (flowRunId != 0L) {
+ collector.getTimelineEntityContext().setFlowRunId(flowRunId);
+ }
+ }
+
+ @VisibleForTesting
+ protected CollectorNodemanagerProtocol getNMCollectorService() {
+ Configuration conf = getConfig();
+ final YarnRPC rpc = YarnRPC.create(conf);
+
+ // TODO Security settings.
+ return (CollectorNodemanagerProtocol) rpc.getProxy(
+ CollectorNodemanagerProtocol.class,
+ nmCollectorServiceAddress, conf);
+ }
+
+ @VisibleForTesting
+ public String getRestServerBindAddress() {
+ return timelineRestServerBindAddress;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 2017d01..36ff5c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -53,15 +53,15 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
LogFactory.getLog(PerNodeTimelineCollectorsAuxService.class);
private static final int SHUTDOWN_HOOK_PRIORITY = 30;
- private final TimelineCollectorManager collectorManager;
+ private final NodeTimelineCollectorManager collectorManager;
public PerNodeTimelineCollectorsAuxService() {
// use the same singleton
- this(TimelineCollectorManager.getInstance());
+ this(NodeTimelineCollectorManager.getInstance());
}
@VisibleForTesting PerNodeTimelineCollectorsAuxService(
- TimelineCollectorManager collectorsManager) {
+ NodeTimelineCollectorManager collectorsManager) {
super("timeline_collector");
this.collectorManager = collectorsManager;
}
@@ -108,8 +108,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
* @return whether it was removed successfully
*/
public boolean removeApplication(ApplicationId appId) {
- String appIdString = appId.toString();
- return collectorManager.remove(appIdString);
+ return collectorManager.remove(appId);
}
/**
@@ -153,8 +152,8 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
}
@VisibleForTesting
- boolean hasApplication(String appId) {
- return collectorManager.containsKey(appId);
+ boolean hasApplication(ApplicationId appId) {
+ return collectorManager.containsTimelineCollector(appId);
}
@Override
@@ -174,7 +173,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
@VisibleForTesting
public static PerNodeTimelineCollectorsAuxService
- launchServer(String[] args, TimelineCollectorManager collectorManager) {
+ launchServer(String[] args, NodeTimelineCollectorManager collectorManager) {
Thread
.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index f1d3d72..4eced5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -124,6 +124,6 @@ public abstract class TimelineCollector extends CompositeService {
}
}
- protected abstract TimelineCollectorContext getTimelineEntityContext();
+ public abstract TimelineCollectorContext getTimelineEntityContext();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 9a566a2..7b3da6b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -18,173 +18,97 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.DEFAULT_HADOOP_HTTP_STATIC_USER;
-import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_HTTP_STATIC_USER;
-
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URI;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.http.HttpServer2;
-import org.apache.hadoop.http.lib.StaticUserWebFilter;
-import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.ipc.YarnRPC;
-import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
-import org.apache.hadoop.yarn.server.api.protocolrecords.ReportNewCollectorInfoRequest;
-import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
-import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
-import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
-import com.google.common.annotations.VisibleForTesting;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
/**
* Class that manages adding and removing collectors and their lifecycle. It
* provides thread safety access to the collectors inside.
*
- * It is a singleton, and instances should be obtained via
- * {@link #getInstance()}.
*/
-@Private
-@Unstable
-public class TimelineCollectorManager extends CompositeService {
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
- private static final TimelineCollectorManager INSTANCE =
- new TimelineCollectorManager();
// access to this map is synchronized with the map itself
- private final Map<String, TimelineCollector> collectors =
+ private final Map<ApplicationId, TimelineCollector> collectors =
Collections.synchronizedMap(
- new HashMap<String, TimelineCollector>());
-
- // REST server for this collector manager
- private HttpServer2 timelineRestServer;
-
- private String timelineRestServerBindAddress;
-
- private CollectorNodemanagerProtocol nmCollectorService;
-
- private InetSocketAddress nmCollectorServiceAddress;
-
- static final String COLLECTOR_MANAGER_ATTR_KEY = "collector.manager";
-
- static TimelineCollectorManager getInstance() {
- return INSTANCE;
- }
+ new HashMap<ApplicationId, TimelineCollector>());
- @VisibleForTesting
- protected TimelineCollectorManager() {
- super(TimelineCollectorManager.class.getName());
- }
-
- @Override
- public void serviceInit(Configuration conf) throws Exception {
- this.nmCollectorServiceAddress = conf.getSocketAddr(
- YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_COLLECTOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_ADDRESS,
- YarnConfiguration.DEFAULT_NM_COLLECTOR_SERVICE_PORT);
-
- }
-
- @Override
- protected void serviceStart() throws Exception {
- nmCollectorService = getNMCollectorService();
- startWebApp();
- super.serviceStart();
- }
-
- @Override
- protected void serviceStop() throws Exception {
- if (timelineRestServer != null) {
- timelineRestServer.stop();
- }
- super.serviceStop();
+ protected TimelineCollectorManager(String name) {
+ super(name);
}
/**
* Put the collector into the collection if an collector mapped by id does
* not exist.
*
- * @throws YarnRuntimeException if there was any exception in initializing and
- * starting the app level service
+ * @throws YarnRuntimeException if there was any exception in initializing
+ * and starting the app level service
* @return the collector associated with id after the potential put.
*/
public TimelineCollector putIfAbsent(ApplicationId appId,
TimelineCollector collector) {
- String id = appId.toString();
- TimelineCollector collectorInTable;
- boolean collectorIsNew = false;
+ TimelineCollector collectorInTable = null;
synchronized (collectors) {
- collectorInTable = collectors.get(id);
+ collectorInTable = collectors.get(appId);
if (collectorInTable == null) {
try {
// initialize, start, and add it to the collection so it can be
// cleaned up when the parent shuts down
collector.init(getConfig());
collector.start();
- collectors.put(id, collector);
- LOG.info("the collector for " + id + " was added");
+ collectors.put(appId, collector);
+ LOG.info("the collector for " + appId + " was added");
collectorInTable = collector;
- collectorIsNew = true;
+ postPut(appId, collectorInTable);
} catch (Exception e) {
throw new YarnRuntimeException(e);
}
} else {
- String msg = "the collector for " + id + " already exists!";
- LOG.error(msg);
- }
-
- }
- // Report to NM if a new collector is added.
- if (collectorIsNew) {
- try {
- updateTimelineCollectorContext(appId, collector);
- reportNewCollectorToNM(appId);
- } catch (Exception e) {
- // throw exception here as it cannot be used if failed communicate with NM
- LOG.error("Failed to communicate with NM Collector Service for " + appId);
- throw new YarnRuntimeException(e);
+ LOG.info("the collector for " + appId + " already exists!");
}
}
-
return collectorInTable;
}
+ protected void postPut(ApplicationId appId, TimelineCollector collector) {
+
+ }
+
/**
* Removes the collector for the specified id. The collector is also stopped
* as a result. If the collector does not exist, no change is made.
*
* @return whether it was removed successfully
*/
- public boolean remove(String id) {
- synchronized (collectors) {
- TimelineCollector collector = collectors.remove(id);
- if (collector == null) {
- String msg = "the collector for " + id + " does not exist!";
- LOG.error(msg);
- return false;
- } else {
- // stop the service to do clean up
- collector.stop();
- LOG.info("the collector service for " + id + " was removed");
- return true;
- }
+ public boolean remove(ApplicationId appId) {
+ TimelineCollector collector = collectors.remove(appId);
+ if (collector == null) {
+ LOG.error("the collector for " + appId + " does not exist!");
+ } else {
+ postRemove(appId, collector);
+ // stop the service to do clean up
+ collector.stop();
+ LOG.info("the collector service for " + appId + " was removed");
}
+ return collector != null;
+ }
+
+ protected void postRemove(ApplicationId appId, TimelineCollector collector) {
+
}
/**
@@ -192,113 +116,16 @@ public class TimelineCollectorManager extends CompositeService {
*
* @return the collector or null if it does not exist
*/
- public TimelineCollector get(String id) {
- return collectors.get(id);
+ public TimelineCollector get(ApplicationId appId) {
+ return collectors.get(appId);
}
/**
* Returns whether the collector for the specified id exists in this
* collection.
*/
- public boolean containsKey(String id) {
- return collectors.containsKey(id);
- }
-
- /**
- * Launch the REST web server for this collector manager
- */
- private void startWebApp() {
- Configuration conf = getConfig();
- String bindAddress = conf.get(YarnConfiguration.TIMELINE_SERVICE_BIND_HOST,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_BIND_HOST) + ":0";
- try {
- Configuration confForInfoServer = new Configuration(conf);
- confForInfoServer.setInt(HttpServer2.HTTP_MAX_THREADS, 10);
- HttpServer2.Builder builder = new HttpServer2.Builder()
- .setName("timeline")
- .setConf(conf)
- .addEndpoint(URI.create(
- (YarnConfiguration.useHttps(conf) ? "https://" : "http://") +
- bindAddress));
- timelineRestServer = builder.build();
- // TODO: replace this by an authentication filter in future.
- HashMap<String, String> options = new HashMap<>();
- String username = conf.get(HADOOP_HTTP_STATIC_USER,
- DEFAULT_HADOOP_HTTP_STATIC_USER);
- options.put(HADOOP_HTTP_STATIC_USER, username);
- HttpServer2.defineFilter(timelineRestServer.getWebAppContext(),
- "static_user_filter_timeline",
- StaticUserWebFilter.StaticUserFilter.class.getName(),
- options, new String[] {"/*"});
-
- timelineRestServer.addJerseyResourcePackage(
- TimelineCollectorWebService.class.getPackage().getName() + ";"
- + GenericExceptionHandler.class.getPackage().getName() + ";"
- + YarnJacksonJaxbJsonProvider.class.getPackage().getName(),
- "/*");
- timelineRestServer.setAttribute(COLLECTOR_MANAGER_ATTR_KEY, this);
- timelineRestServer.start();
- } catch (Exception e) {
- String msg = "The per-node collector webapp failed to start.";
- LOG.error(msg, e);
- throw new YarnRuntimeException(msg, e);
- }
- //TODO: We need to think of the case of multiple interfaces
- this.timelineRestServerBindAddress = WebAppUtils.getResolvedAddress(
- timelineRestServer.getConnectorAddress(0));
- LOG.info("Instantiated the per-node collector webapp at " +
- timelineRestServerBindAddress);
- }
-
- private void reportNewCollectorToNM(ApplicationId appId)
- throws YarnException, IOException {
- ReportNewCollectorInfoRequest request =
- ReportNewCollectorInfoRequest.newInstance(appId,
- this.timelineRestServerBindAddress);
- LOG.info("Report a new collector for application: " + appId +
- " to the NM Collector Service.");
- nmCollectorService.reportNewCollectorInfo(request);
+ public boolean containsTimelineCollector(ApplicationId appId) {
+ return collectors.containsKey(appId);
}
- private void updateTimelineCollectorContext(
- ApplicationId appId, TimelineCollector collector)
- throws YarnException, IOException {
- GetTimelineCollectorContextRequest request =
- GetTimelineCollectorContextRequest.newInstance(appId);
- LOG.info("Get timeline collector context for " + appId);
- GetTimelineCollectorContextResponse response =
- nmCollectorService.getTimelineCollectorContext(request);
- String userId = response.getUserId();
- if (userId != null && !userId.isEmpty()) {
- collector.getTimelineEntityContext().setUserId(userId);
- }
- String flowName = response.getFlowName();
- if (flowName != null && !flowName.isEmpty()) {
- collector.getTimelineEntityContext().setFlowName(flowName);
- }
- String flowVersion = response.getFlowVersion();
- if (flowVersion != null && !flowVersion.isEmpty()) {
- collector.getTimelineEntityContext().setFlowVersion(flowVersion);
- }
- long flowRunId = response.getFlowRunId();
- if (flowRunId != 0L) {
- collector.getTimelineEntityContext().setFlowRunId(flowRunId);
- }
- }
-
- @VisibleForTesting
- protected CollectorNodemanagerProtocol getNMCollectorService() {
- Configuration conf = getConfig();
- final YarnRPC rpc = YarnRPC.create(conf);
-
- // TODO Security settings.
- return (CollectorNodemanagerProtocol) rpc.getProxy(
- CollectorNodemanagerProtocol.class,
- nmCollectorServiceAddress, conf);
- }
-
- @VisibleForTesting
- public String getRestServerBindAddress() {
- return timelineRestServerBindAddress;
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index edec0d3..2165c66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.ForbiddenException;
@@ -129,11 +130,14 @@ public class TimelineCollectorWebService {
boolean isAsync = async != null && async.trim().equalsIgnoreCase("true");
try {
- appId = parseApplicationId(appId);
- if (appId == null) {
+ ApplicationId appID = parseApplicationId(appId);
+ if (appID == null) {
return Response.status(Response.Status.BAD_REQUEST).build();
}
- TimelineCollector collector = getCollector(req, appId);
+ NodeTimelineCollectorManager collectorManager =
+ (NodeTimelineCollectorManager) context.getAttribute(
+ NodeTimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY);
+ TimelineCollector collector = collectorManager.get(appID);
if (collector == null) {
LOG.error("Application: "+ appId + " is not found");
throw new NotFoundException(); // different exception?
@@ -147,10 +151,10 @@ public class TimelineCollectorWebService {
}
}
- private String parseApplicationId(String appId) {
+ private ApplicationId parseApplicationId(String appId) {
try {
if (appId != null) {
- return ConverterUtils.toApplicationId(appId.trim()).toString();
+ return ConverterUtils.toApplicationId(appId.trim());
} else {
return null;
}
@@ -159,15 +163,6 @@ public class TimelineCollectorWebService {
}
}
- private TimelineCollector
- getCollector(HttpServletRequest req, String appIdToParse) {
- String appIdString = parseApplicationId(appIdToParse);
- final TimelineCollectorManager collectorManager =
- (TimelineCollectorManager) context.getAttribute(
- TimelineCollectorManager.COLLECTOR_MANAGER_ATTR_KEY);
- return collectorManager.get(appIdString);
- }
-
private void init(HttpServletResponse response) {
response.setContentType(null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
new file mode 100644
index 0000000..87343fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestNMTimelineCollectorManager.java
@@ -0,0 +1,160 @@
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.doReturn;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestNMTimelineCollectorManager {
+ private NodeTimelineCollectorManager collectorManager;
+
+ @Before
+ public void setup() throws Exception {
+ collectorManager = createCollectorManager();
+ collectorManager.init(new YarnConfiguration());
+ collectorManager.start();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (collectorManager != null) {
+ collectorManager.stop();
+ }
+ }
+
+ @Test
+ public void testStartWebApp() throws Exception {
+ assertNotNull(collectorManager.getRestServerBindAddress());
+ String address = collectorManager.getRestServerBindAddress();
+ String[] parts = address.split(":");
+ assertEquals(2, parts.length);
+ assertNotNull(parts[0]);
+ assertTrue(Integer.valueOf(parts[1]) > 0);
+ }
+
+ @Test(timeout=60000)
+ public void testMultithreadedAdd() throws Exception {
+ final int NUM_APPS = 5;
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ Callable<Boolean> task = new Callable<Boolean>() {
+ public Boolean call() {
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ return (collectorManager.putIfAbsent(appId, collector) == collector);
+ }
+ };
+ tasks.add(task);
+ }
+ ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
+ try {
+ List<Future<Boolean>> futures = executor.invokeAll(tasks);
+ for (Future<Boolean> future: futures) {
+ assertTrue(future.get());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ // check the keys
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ assertTrue(collectorManager.containsTimelineCollector(appId));
+ }
+ }
+
+ @Test
+ public void testMultithreadedAddAndRemove() throws Exception {
+ final int NUM_APPS = 5;
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ Callable<Boolean> task = new Callable<Boolean>() {
+ public Boolean call() {
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ boolean successPut =
+ (collectorManager.putIfAbsent(appId, collector) == collector);
+ return successPut && collectorManager.remove(appId);
+ }
+ };
+ tasks.add(task);
+ }
+ ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
+ try {
+ List<Future<Boolean>> futures = executor.invokeAll(tasks);
+ for (Future<Boolean> future: futures) {
+ assertTrue(future.get());
+ }
+ } finally {
+ executor.shutdownNow();
+ }
+ // check the keys
+ for (int i = 0; i < NUM_APPS; i++) {
+ final ApplicationId appId = ApplicationId.newInstance(0L, i);
+ assertFalse(collectorManager.containsTimelineCollector(appId));
+ }
+ }
+
+ private NodeTimelineCollectorManager createCollectorManager() {
+ final NodeTimelineCollectorManager collectorManager =
+ spy(new NodeTimelineCollectorManager());
+ doReturn(new Configuration()).when(collectorManager).getConfig();
+ CollectorNodemanagerProtocol nmCollectorService =
+ mock(CollectorNodemanagerProtocol.class);
+ GetTimelineCollectorContextResponse response =
+ GetTimelineCollectorContextResponse.newInstance(null, null, null, 0L);
+ try {
+ when(nmCollectorService.getTimelineCollectorContext(any(
+ GetTimelineCollectorContextRequest.class))).thenReturn(response);
+ } catch (YarnException | IOException e) {
+ fail();
+ }
+ doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
+ return collectorManager;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index abbe13a..b1a5b04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -67,8 +67,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
public void testAddApplication() throws Exception {
auxService = createCollectorAndAddApplication();
// auxService should have a single app
- assertTrue(auxService.hasApplication(
- appAttemptId.getApplicationId().toString()));
+ assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
auxService.close();
}
@@ -82,16 +81,14 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerId()).thenReturn(containerId);
auxService.initializeContainer(context);
// auxService should not have that app
- assertFalse(auxService.hasApplication(
- appAttemptId.getApplicationId().toString()));
+ assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
}
@Test
public void testRemoveApplication() throws Exception {
auxService = createCollectorAndAddApplication();
// auxService should have a single app
- String appIdStr = appAttemptId.getApplicationId().toString();
- assertTrue(auxService.hasApplication(appIdStr));
+ assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
ContainerId containerId = getAMContainerId();
ContainerTerminationContext context =
@@ -99,7 +96,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerId()).thenReturn(containerId);
auxService.stopContainer(context);
// auxService should not have that app
- assertFalse(auxService.hasApplication(appIdStr));
+ assertFalse(auxService.hasApplication(appAttemptId.getApplicationId()));
auxService.close();
}
@@ -107,8 +104,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
public void testRemoveApplicationNonAMContainer() throws Exception {
auxService = createCollectorAndAddApplication();
// auxService should have a single app
- String appIdStr = appAttemptId.getApplicationId().toString();
- assertTrue(auxService.hasApplication(appIdStr));
+ assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
ContainerId containerId = getContainerId(2L); // not an AM
ContainerTerminationContext context =
@@ -116,7 +112,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerId()).thenReturn(containerId);
auxService.stopContainer(context);
// auxService should still have that app
- assertTrue(auxService.hasApplication(appIdStr));
+ assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
auxService.close();
}
@@ -147,7 +143,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
}
private PerNodeTimelineCollectorsAuxService createCollector() {
- TimelineCollectorManager collectorManager = createCollectorManager();
+ NodeTimelineCollectorManager collectorManager = createCollectorManager();
PerNodeTimelineCollectorsAuxService auxService =
spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
auxService.init(new YarnConfiguration());
@@ -155,9 +151,9 @@ public class TestPerNodeTimelineCollectorsAuxService {
return auxService;
}
- private TimelineCollectorManager createCollectorManager() {
- TimelineCollectorManager collectorManager =
- spy(new TimelineCollectorManager());
+ private NodeTimelineCollectorManager createCollectorManager() {
+ NodeTimelineCollectorManager collectorManager =
+ spy(new NodeTimelineCollectorManager());
doReturn(new Configuration()).when(collectorManager).getConfig();
CollectorNodemanagerProtocol nmCollectorService =
mock(CollectorNodemanagerProtocol.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9e8d853/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
deleted file mode 100644
index c662998..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollectorManager.java
+++ /dev/null
@@ -1,160 +0,0 @@
-
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.collector;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.when;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
-import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
-import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-
-public class TestTimelineCollectorManager {
- private TimelineCollectorManager collectorManager;
-
- @Before
- public void setup() throws Exception {
- collectorManager = createCollectorManager();
- collectorManager.init(new YarnConfiguration());
- collectorManager.start();
- }
-
- @After
- public void tearDown() throws Exception {
- if (collectorManager != null) {
- collectorManager.stop();
- }
- }
-
- @Test
- public void testStartWebApp() throws Exception {
- assertNotNull(collectorManager.getRestServerBindAddress());
- String address = collectorManager.getRestServerBindAddress();
- String[] parts = address.split(":");
- assertEquals(2, parts.length);
- assertNotNull(parts[0]);
- assertTrue(Integer.valueOf(parts[1]) > 0);
- }
-
- @Test(timeout=60000)
- public void testMultithreadedAdd() throws Exception {
- final int NUM_APPS = 5;
- List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
- for (int i = 0; i < NUM_APPS; i++) {
- final ApplicationId appId = ApplicationId.newInstance(0L, i);
- Callable<Boolean> task = new Callable<Boolean>() {
- public Boolean call() {
- AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId);
- return (collectorManager.putIfAbsent(appId, collector) == collector);
- }
- };
- tasks.add(task);
- }
- ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
- try {
- List<Future<Boolean>> futures = executor.invokeAll(tasks);
- for (Future<Boolean> future: futures) {
- assertTrue(future.get());
- }
- } finally {
- executor.shutdownNow();
- }
- // check the keys
- for (int i = 0; i < NUM_APPS; i++) {
- final ApplicationId appId = ApplicationId.newInstance(0L, i);
- assertTrue(collectorManager.containsKey(appId.toString()));
- }
- }
-
- @Test
- public void testMultithreadedAddAndRemove() throws Exception {
- final int NUM_APPS = 5;
- List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>();
- for (int i = 0; i < NUM_APPS; i++) {
- final ApplicationId appId = ApplicationId.newInstance(0L, i);
- Callable<Boolean> task = new Callable<Boolean>() {
- public Boolean call() {
- AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId);
- boolean successPut =
- (collectorManager.putIfAbsent(appId, collector) == collector);
- return successPut && collectorManager.remove(appId.toString());
- }
- };
- tasks.add(task);
- }
- ExecutorService executor = Executors.newFixedThreadPool(NUM_APPS);
- try {
- List<Future<Boolean>> futures = executor.invokeAll(tasks);
- for (Future<Boolean> future: futures) {
- assertTrue(future.get());
- }
- } finally {
- executor.shutdownNow();
- }
- // check the keys
- for (int i = 0; i < NUM_APPS; i++) {
- final ApplicationId appId = ApplicationId.newInstance(0L, i);
- assertFalse(collectorManager.containsKey(appId.toString()));
- }
- }
-
- private TimelineCollectorManager createCollectorManager() {
- final TimelineCollectorManager collectorManager =
- spy(new TimelineCollectorManager());
- doReturn(new Configuration()).when(collectorManager).getConfig();
- CollectorNodemanagerProtocol nmCollectorService =
- mock(CollectorNodemanagerProtocol.class);
- GetTimelineCollectorContextResponse response =
- GetTimelineCollectorContextResponse.newInstance(null, null, null, 0L);
- try {
- when(nmCollectorService.getTimelineCollectorContext(any(
- GetTimelineCollectorContextRequest.class))).thenReturn(response);
- } catch (YarnException | IOException e) {
- fail();
- }
- doReturn(nmCollectorService).when(collectorManager).getNMCollectorService();
- return collectorManager;
- }
-}
[24/50] [abbrv] hadoop git commit: YARN-3044. Made RM write app,
attempt and optional container lifecycle events to timeline service
v2. Contributed by Naganarasimha G R.
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
new file mode 100644
index 0000000..9830a80
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.AsyncDispatcher;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.AbstractTimelineServicePublisher.MultiThreadedDispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestSystemMetricsPublisherForV2 {
+
+ /**
+ * is the folder where the FileSystemTimelineWriterImpl writes the entities
+ */
+ protected static File testRootDir = new File("target",
+ TestSystemMetricsPublisherForV2.class.getName() + "-localDir")
+ .getAbsoluteFile();
+
+ private static SystemMetricsPublisher metricsPublisher;
+ private static DrainDispatcher dispatcher = new DrainDispatcher();
+ private static final String DEFAULT_FLOW_VERSION = "1";
+ private static final long DEFAULT_FLOW_RUN = 1;
+
+ private static ConcurrentMap<ApplicationId, RMApp> rmAppsMapInContext;
+
+ private static RMTimelineCollectorManager rmTimelineCollectorManager;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ if (testRootDir.exists()) {
+ //cleanup before hand
+ FileContext.getLocalFSFileContext().delete(
+ new Path(testRootDir.getAbsolutePath()), true);
+ }
+
+ RMContext rmContext = mock(RMContext.class);
+ rmAppsMapInContext = new ConcurrentHashMap<ApplicationId, RMApp>();
+ when(rmContext.getRMApps()).thenReturn(rmAppsMapInContext);
+ rmTimelineCollectorManager = new RMTimelineCollectorManager(rmContext);
+ when(rmContext.getRMTimelineCollectorManager()).thenReturn(
+ rmTimelineCollectorManager);
+
+ Configuration conf = getTimelineV2Conf();
+ rmTimelineCollectorManager.init(conf);
+ rmTimelineCollectorManager.start();
+
+ metricsPublisher = new SystemMetricsPublisher(rmContext) {
+ @Override
+ Dispatcher createDispatcher(
+ TimelineServicePublisher timelineServicePublisher) {
+ return dispatcher;
+ }
+ };
+ metricsPublisher.init(conf);
+ metricsPublisher.start();
+ }
+
+ @AfterClass
+ public static void tearDown() throws Exception {
+ if (testRootDir.exists()) {
+ FileContext.getLocalFSFileContext().delete(
+ new Path(testRootDir.getAbsolutePath()), true);
+ }
+ if (rmTimelineCollectorManager != null) {
+ rmTimelineCollectorManager.stop();
+ }
+ if (metricsPublisher != null) {
+ metricsPublisher.stop();
+ }
+ }
+
+ private static Configuration getTimelineV2Conf() {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, 2);
+ conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ true);
+ try {
+ conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
+ testRootDir.getCanonicalPath());
+ } catch (IOException e) {
+ e.printStackTrace();
+ Assert
+ .fail("Exception while setting the TIMELINE_SERVICE_STORAGE_DIR_ROOT ");
+ }
+ return conf;
+ }
+
+ @Test
+ public void testSystemMetricPublisherInitialization() {
+ @SuppressWarnings("resource")
+ SystemMetricsPublisher metricsPublisher =
+ new SystemMetricsPublisher(mock(RMContext.class));
+ try {
+ Configuration conf = getTimelineV2Conf();
+ conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ false);
+ metricsPublisher.init(conf);
+ assertFalse(
+ "Default configuration should not publish container Metrics from RM",
+ metricsPublisher.isPublishContainerMetrics());
+
+ metricsPublisher.stop();
+
+ metricsPublisher = new SystemMetricsPublisher(mock(RMContext.class));
+ conf = getTimelineV2Conf();
+ metricsPublisher.init(conf);
+ assertTrue("Expected to publish container Metrics from RM",
+ metricsPublisher.isPublishContainerMetrics());
+ assertTrue(
+ "MultiThreadedDispatcher expected when container Metrics is not published",
+ metricsPublisher.getDispatcher() instanceof MultiThreadedDispatcher);
+ } finally {
+ metricsPublisher.stop();
+ }
+ }
+
+ @Test(timeout = 1000000)
+ public void testPublishApplicationMetrics() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ RMApp app = createAppAndRegister(appId);
+
+ metricsPublisher.appCreated(app, app.getStartTime());
+ metricsPublisher.appACLsUpdated(app, "user1,user2", 4L);
+ metricsPublisher.appFinished(app, RMAppState.FINISHED, app.getFinishTime());
+ dispatcher.await();
+
+ String outputDirApp =
+ getTimelineEntityDir(app) + "/" + TimelineEntityType.YARN_APPLICATION
+ + "/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // file name is <entityId>.thist
+ String timelineServiceFileName =
+ appId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appFile = new File(outputDirApp, timelineServiceFileName);
+ Assert.assertTrue(appFile.exists());
+ Assert.assertEquals("Expected 3 events to be published", 3,
+ getNumOfNonEmptyLines(appFile));
+ }
+
+ @Test(timeout = 10000)
+ public void testPublishAppAttemptMetrics() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ RMApp app = rmAppsMapInContext.get(appId);
+ if (app == null) {
+ app = createAppAndRegister(appId);
+ }
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 1);
+ RMAppAttempt appAttempt = createRMAppAttempt(appAttemptId);
+ metricsPublisher.appAttemptRegistered(appAttempt, Integer.MAX_VALUE + 1L);
+ when(app.getFinalApplicationStatus()).thenReturn(
+ FinalApplicationStatus.UNDEFINED);
+ metricsPublisher.appAttemptFinished(appAttempt, RMAppAttemptState.FINISHED,
+ app, Integer.MAX_VALUE + 2L);
+
+ dispatcher.await();
+
+ String outputDirApp =
+ getTimelineEntityDir(app) + "/"
+ + TimelineEntityType.YARN_APPLICATION_ATTEMPT + "/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // file name is <entityId>.thist
+ String timelineServiceFileName =
+ appAttemptId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appFile = new File(outputDirApp, timelineServiceFileName);
+ Assert.assertTrue(appFile.exists());
+ Assert.assertEquals("Expected 2 events to be published", 2,
+ getNumOfNonEmptyLines(appFile));
+ }
+
+ @Test(timeout = 10000)
+ public void testPublishContainerMetrics() throws Exception {
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ RMApp app = rmAppsMapInContext.get(appId);
+ if (app == null) {
+ app = createAppAndRegister(appId);
+ }
+ ContainerId containerId =
+ ContainerId.newContainerId(ApplicationAttemptId.newInstance(
+ appId, 1), 1);
+ RMContainer container = createRMContainer(containerId);
+ metricsPublisher.containerCreated(container, container.getCreationTime());
+ metricsPublisher.containerFinished(container, container.getFinishTime());
+ dispatcher.await();
+
+ String outputDirApp =
+ getTimelineEntityDir(app) + "/"
+ + TimelineEntityType.YARN_CONTAINER + "/";
+
+ File entityFolder = new File(outputDirApp);
+ Assert.assertTrue(entityFolder.isDirectory());
+
+ // file name is <entityId>.thist
+ String timelineServiceFileName =
+ containerId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ File appFile = new File(outputDirApp, timelineServiceFileName);
+ Assert.assertTrue(appFile.exists());
+ Assert.assertEquals("Expected 2 events to be published", 2,
+ getNumOfNonEmptyLines(appFile));
+ }
+
+ private RMApp createAppAndRegister(ApplicationId appId) {
+ RMApp app = createRMApp(appId);
+
+ // some stuff which are currently taken care in RMAppImpl
+ rmAppsMapInContext.putIfAbsent(appId, app);
+ AppLevelTimelineCollector collector = new AppLevelTimelineCollector(appId);
+ rmTimelineCollectorManager.putIfAbsent(appId, collector);
+ return app;
+ }
+
+ private long getNumOfNonEmptyLines(File entityFile) throws IOException {
+ BufferedReader reader = null;
+ String strLine;
+ long count = 0;
+ try {
+ reader = new BufferedReader(new FileReader(entityFile));
+ while ((strLine = reader.readLine()) != null) {
+ if (strLine.trim().length() > 0)
+ count++;
+ }
+ } finally {
+ reader.close();
+ }
+ return count;
+ }
+
+ private String getTimelineEntityDir(RMApp app) {
+ String outputDirApp =
+ testRootDir.getAbsolutePath()+"/"
+ + FileSystemTimelineWriterImpl.ENTITIES_DIR
+ + "/"
+ + YarnConfiguration.DEFAULT_RM_CLUSTER_ID
+ + "/"
+ + app.getUser()
+ + "/"
+ + TimelineUtils.generateDefaultFlowIdBasedOnAppId(app
+ .getApplicationId()) + "/" + DEFAULT_FLOW_VERSION + "/"
+ + DEFAULT_FLOW_RUN + "/" + app.getApplicationId();
+ return outputDirApp;
+ }
+
+ private static RMApp createRMApp(ApplicationId appId) {
+ RMApp app = mock(RMAppImpl.class);
+ when(app.getApplicationId()).thenReturn(appId);
+ when(app.getName()).thenReturn("test app");
+ when(app.getApplicationType()).thenReturn("test app type");
+ when(app.getUser()).thenReturn("testUser");
+ when(app.getQueue()).thenReturn("test queue");
+ when(app.getSubmitTime()).thenReturn(Integer.MAX_VALUE + 1L);
+ when(app.getStartTime()).thenReturn(Integer.MAX_VALUE + 2L);
+ when(app.getFinishTime()).thenReturn(Integer.MAX_VALUE + 3L);
+ when(app.getDiagnostics()).thenReturn(
+ new StringBuilder("test diagnostics info"));
+ RMAppAttempt appAttempt = mock(RMAppAttempt.class);
+ when(appAttempt.getAppAttemptId()).thenReturn(
+ ApplicationAttemptId.newInstance(appId, 1));
+ when(app.getCurrentAppAttempt()).thenReturn(appAttempt);
+ when(app.getFinalApplicationStatus()).thenReturn(
+ FinalApplicationStatus.UNDEFINED);
+ when(app.getRMAppMetrics()).thenReturn(
+ new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
+ when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
+ return app;
+ }
+
+ private static RMAppAttempt createRMAppAttempt(
+ ApplicationAttemptId appAttemptId) {
+ RMAppAttempt appAttempt = mock(RMAppAttempt.class);
+ when(appAttempt.getAppAttemptId()).thenReturn(appAttemptId);
+ when(appAttempt.getHost()).thenReturn("test host");
+ when(appAttempt.getRpcPort()).thenReturn(-100);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(
+ ContainerId.newContainerId(appAttemptId, 1));
+ when(appAttempt.getMasterContainer()).thenReturn(container);
+ when(appAttempt.getDiagnostics()).thenReturn("test diagnostics info");
+ when(appAttempt.getTrackingUrl()).thenReturn("test tracking url");
+ when(appAttempt.getOriginalTrackingUrl()).thenReturn(
+ "test original tracking url");
+ return appAttempt;
+ }
+
+ private static RMContainer createRMContainer(ContainerId containerId) {
+ RMContainer container = mock(RMContainer.class);
+ when(container.getContainerId()).thenReturn(containerId);
+ when(container.getAllocatedNode()).thenReturn(
+ NodeId.newInstance("test host", -100));
+ when(container.getAllocatedResource()).thenReturn(
+ Resource.newInstance(-1, -1));
+ when(container.getAllocatedPriority()).thenReturn(Priority.UNDEFINED);
+ when(container.getCreationTime()).thenReturn(Integer.MAX_VALUE + 1L);
+ when(container.getFinishTime()).thenReturn(Integer.MAX_VALUE + 2L);
+ when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info");
+ when(container.getContainerExitStatus()).thenReturn(-1);
+ when(container.getContainerState()).thenReturn(ContainerState.COMPLETE);
+ Container mockContainer = mock(Container.class);
+ when(container.getContainer()).thenReturn(mockContainer);
+ when(mockContainer.getNodeHttpAddress())
+ .thenReturn("http://localhost:1234");
+ return container;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a22d26c6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index ee1515d..34a6b7c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -55,7 +55,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
= "/tmp/timeline_service_data";
- private static final String ENTITIES_DIR = "entities";
+ public static final String ENTITIES_DIR = "entities";
/** Default extension for output files */
public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
@@ -76,7 +76,7 @@ public class FileSystemTimelineWriterImpl extends AbstractService
return response;
}
- private void write(String clusterId, String userId, String flowName,
+ private synchronized void write(String clusterId, String userId, String flowName,
String flowVersion, long flowRun, String appId, TimelineEntity entity,
TimelineWriteResponse response) throws IOException {
PrintWriter out = null;
[26/50] [abbrv] hadoop git commit: YARN-3721. build is broken on
YARN-2928 branch due to possible dependency cycle (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-3721. build is broken on YARN-2928 branch due to possible dependency cycle (Li Lu via sjlee)
(cherry picked from commit a9738ceb17b50cce8844fd42bb800c7f83f15caf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/534aae59
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/534aae59
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/534aae59
Branch: refs/heads/YARN-2928
Commit: 534aae59ac8328edf75dadca99b06f6c4ecddc39
Parents: d0d1cc0
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu May 28 12:03:53 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:12 2015 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 97 ++++++++++----------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop-yarn-server-timelineservice/pom.xml | 1 -
3 files changed, 53 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/534aae59/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 7a35939..1182d6d 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -996,55 +996,58 @@
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-client</artifactId>
- <version>${hbase.version}</version>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <version>${phoenix.version}</version>
- <exclusions>
- <!-- Exclude jline from here -->
- <exclusion>
- <artifactId>jline</artifactId>
- <groupId>jline</groupId>
- </exclusion>
- </exclusions>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-core</artifactId>
- <type>test-jar</type>
- <version>${phoenix.version}</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-it</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- <classifier>tests</classifier>
- </dependency>
- <dependency>
- <groupId>org.apache.hbase</groupId>
- <artifactId>hbase-testing-util</artifactId>
- <version>${hbase.version}</version>
- <scope>test</scope>
- <optional>true</optional>
- <exclusions>
- <exclusion>
- <groupId>org.jruby</groupId>
- <artifactId>jruby-complete</artifactId>
- </exclusion>
- <exclusion>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-client</artifactId>
+ <version>${hbase.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <version>${phoenix.version}</version>
+ <exclusions>
+ <!-- Exclude jline from here -->
+ <exclusion>
+ <artifactId>jline</artifactId>
+ <groupId>jline</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-core</artifactId>
+ <type>test-jar</type>
+ <version>${phoenix.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-it</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <classifier>tests</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-testing-util</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ <optional>true</optional>
+ <exclusions>
+ <exclusion>
+ <groupId>org.jruby</groupId>
+ <artifactId>jruby-complete</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- </exclusion>
- </exclusions>
- </dependency>
-
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</dependencyManagement>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/534aae59/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bc3880c..5d8aefc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -79,6 +79,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3726. Fix TestHBaseTimelineWriterImpl unit test failure by fixing its
test data (Vrushali C via sjlee)
+ YARN-3721. build is broken on YARN-2928 branch due to possible dependency
+ cycle (Li Lu via sjlee)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/534aae59/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 1e914de..da7fadf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -159,7 +159,6 @@
<scope>test</scope>
<optional>true</optional>
</dependency>
-
</dependencies>
<build>
[45/50] [abbrv] hadoop git commit: MAPREDUCE-6337. Added a mode to
replay MR job history files and put them into the timeline service v2.
Contributed by Sangjin Lee.
Posted by sj...@apache.org.
MAPREDUCE-6337. Added a mode to replay MR job history files and put them into the timeline service v2. Contributed by Sangjin Lee.
(cherry picked from commit 463e070a8e7c882706a96eaa20ea49bfe9982875)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/09a8b7b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/09a8b7b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/09a8b7b9
Branch: refs/heads/YARN-2928
Commit: 09a8b7b94b08572c3e279a9de3103529acbccc1f
Parents: 51029a7
Author: Zhijie Shen <zj...@apache.org>
Authored: Thu May 14 15:16:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:16 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../hadoop/mapred/JobHistoryFileParser.java | 53 ++++
.../mapred/JobHistoryFileReplayMapper.java | 301 +++++++++++++++++++
.../hadoop/mapred/SimpleEntityWriter.java | 139 +++++++++
.../hadoop/mapred/TimelineEntityConverter.java | 207 +++++++++++++
.../mapred/TimelineServicePerformanceV2.java | 191 ++++--------
.../collector/TimelineCollectorManager.java | 8 +-
.../storage/FileSystemTimelineWriterImpl.java | 23 +-
.../timelineservice/storage/package-info.java | 24 ++
9 files changed, 809 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 2805780..9c66a5e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
MAPREDUCE-6335. Created MR job based performance test driver for the
timeline service v2. (Sangjin Lee via zjshen)
+ MAPREDUCE-6337. Added a mode to replay MR job history files and put them
+ into the timeline service v2. (Sangjin Lee via zjshen)
+
IMPROVEMENTS
OPTIMIZATIONS
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
new file mode 100644
index 0000000..9d051df
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
@@ -0,0 +1,53 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+
+class JobHistoryFileParser {
+ private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
+
+ private final FileSystem fs;
+
+ public JobHistoryFileParser(FileSystem fs) {
+ LOG.info("JobHistoryFileParser created with " + fs);
+ this.fs = fs;
+ }
+
+ public JobInfo parseHistoryFile(Path path) throws IOException {
+ LOG.info("parsing job history file " + path);
+ JobHistoryParser parser = new JobHistoryParser(fs, path);
+ return parser.parse();
+ }
+
+ public Configuration parseConfiguration(Path path) throws IOException {
+ LOG.info("parsing job configuration file " + path);
+ Configuration conf = new Configuration(false);
+ conf.addResource(fs.open(path));
+ return conf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
new file mode 100644
index 0000000..802b78f
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
@@ -0,0 +1,301 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.EntityWriter;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.PerfCounters;
+import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Mapper for TimelineServicePerformanceV2 that replays job history files to the
+ * timeline service.
+ *
+ */
+class JobHistoryFileReplayMapper extends EntityWriter {
+ private static final Log LOG =
+ LogFactory.getLog(JobHistoryFileReplayMapper.class);
+
+ static final String PROCESSING_PATH = "processing path";
+ static final String REPLAY_MODE = "replay mode";
+ static final int WRITE_ALL_AT_ONCE = 1;
+ static final int WRITE_PER_ENTITY = 2;
+ static final int REPLAY_MODE_DEFAULT = WRITE_ALL_AT_ONCE;
+
+ private static final Pattern JOB_ID_PARSER =
+ Pattern.compile("^(job_[0-9]+_([0-9]+)).*");
+
+ public static class JobFiles {
+ private final String jobId;
+ private Path jobHistoryFilePath;
+ private Path jobConfFilePath;
+
+ public JobFiles(String jobId) {
+ this.jobId = jobId;
+ }
+
+ public String getJobId() {
+ return jobId;
+ }
+
+ public Path getJobHistoryFilePath() {
+ return jobHistoryFilePath;
+ }
+
+ public void setJobHistoryFilePath(Path jobHistoryFilePath) {
+ this.jobHistoryFilePath = jobHistoryFilePath;
+ }
+
+ public Path getJobConfFilePath() {
+ return jobConfFilePath;
+ }
+
+ public void setJobConfFilePath(Path jobConfFilePath) {
+ this.jobConfFilePath = jobConfFilePath;
+ }
+
+ @Override
+ public int hashCode() {
+ return jobId.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ JobFiles other = (JobFiles) obj;
+ return jobId.equals(other.jobId);
+ }
+ }
+
+ private enum FileType { JOB_HISTORY_FILE, JOB_CONF_FILE, UNKNOWN }
+
+
+ @Override
+ protected void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException {
+ // collect the apps it needs to process
+ Configuration conf = context.getConfiguration();
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ int size = conf.getInt(MRJobConfig.NUM_MAPS,
+ TimelineServicePerformanceV2.NUM_MAPS_DEFAULT);
+ String processingDir =
+ conf.get(JobHistoryFileReplayMapper.PROCESSING_PATH);
+ int replayMode =
+ conf.getInt(JobHistoryFileReplayMapper.REPLAY_MODE,
+ JobHistoryFileReplayMapper.REPLAY_MODE_DEFAULT);
+ Path processingPath = new Path(processingDir);
+ FileSystem processingFs = processingPath.getFileSystem(conf);
+ JobHistoryFileParser parser = new JobHistoryFileParser(processingFs);
+ TimelineEntityConverter converter = new TimelineEntityConverter();
+
+ Collection<JobFiles> jobs =
+ selectJobFiles(processingFs, processingPath, taskId, size);
+ if (jobs.isEmpty()) {
+ LOG.info(context.getTaskAttemptID().getTaskID() +
+ " will process no jobs");
+ } else {
+ LOG.info(context.getTaskAttemptID().getTaskID() + " will process " +
+ jobs.size() + " jobs");
+ }
+ for (JobFiles job: jobs) {
+ // process each job
+ String jobIdStr = job.getJobId();
+ LOG.info("processing " + jobIdStr + "...");
+ JobId jobId = TypeConverter.toYarn(JobID.forName(jobIdStr));
+ ApplicationId appId = jobId.getAppId();
+
+ // create the app level timeline collector and start it
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ manager.putIfAbsent(appId, collector);
+ try {
+ // parse the job info and configuration
+ JobInfo jobInfo =
+ parser.parseHistoryFile(job.getJobHistoryFilePath());
+ Configuration jobConf =
+ parser.parseConfiguration(job.getJobConfFilePath());
+ LOG.info("parsed the job history file and the configuration file for job"
+ + jobIdStr);
+
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(jobInfo.getJobname());
+ tlContext.setFlowRunId(jobInfo.getSubmitTime());
+ tlContext.setUserId(jobInfo.getUsername());
+
+ // create entities from job history and write them
+ long totalTime = 0;
+ Set<TimelineEntity> entitySet =
+ converter.createTimelineEntities(jobInfo, jobConf);
+ LOG.info("converted them into timeline entities for job " + jobIdStr);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ switch (replayMode) {
+ case JobHistoryFileReplayMapper.WRITE_ALL_AT_ONCE:
+ writeAllEntities(collector, entitySet, ugi);
+ break;
+ case JobHistoryFileReplayMapper.WRITE_PER_ENTITY:
+ writePerEntity(collector, entitySet, ugi);
+ break;
+ default:
+ break;
+ }
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ LOG.error("writing to the timeline service failed", e);
+ }
+ long endWrite = System.nanoTime();
+ totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
+ int numEntities = entitySet.size();
+ LOG.info("wrote " + numEntities + " entities in " + totalTime + " ms");
+
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(numEntities);
+ } finally {
+ manager.remove(appId);
+ context.progress(); // move it along
+ }
+ }
+ }
+
+ private void writeAllEntities(AppLevelTimelineCollector collector,
+ Set<TimelineEntity> entitySet, UserGroupInformation ugi)
+ throws IOException {
+ TimelineEntities entities = new TimelineEntities();
+ entities.setEntities(entitySet);
+ collector.putEntities(entities, ugi);
+ }
+
+ private void writePerEntity(AppLevelTimelineCollector collector,
+ Set<TimelineEntity> entitySet, UserGroupInformation ugi)
+ throws IOException {
+ for (TimelineEntity entity : entitySet) {
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ collector.putEntities(entities, ugi);
+ LOG.info("wrote entity " + entity.getId());
+ }
+ }
+
+ private Collection<JobFiles> selectJobFiles(FileSystem fs,
+ Path processingRoot, int i, int size) throws IOException {
+ Map<String,JobFiles> jobs = new HashMap<>();
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(processingRoot, true);
+ while (it.hasNext()) {
+ LocatedFileStatus status = it.next();
+ Path path = status.getPath();
+ String fileName = path.getName();
+ Matcher m = JOB_ID_PARSER.matcher(fileName);
+ if (!m.matches()) {
+ continue;
+ }
+ String jobId = m.group(1);
+ int lastId = Integer.parseInt(m.group(2));
+ int mod = lastId % size;
+ if (mod != i) {
+ continue;
+ }
+ LOG.info("this mapper will process file " + fileName);
+ // it's mine
+ JobFiles jobFiles = jobs.get(jobId);
+ if (jobFiles == null) {
+ jobFiles = new JobFiles(jobId);
+ jobs.put(jobId, jobFiles);
+ }
+ setFilePath(fileName, path, jobFiles);
+ }
+ return jobs.values();
+ }
+
+ private void setFilePath(String fileName, Path path,
+ JobFiles jobFiles) {
+ // determine if we're dealing with a job history file or a job conf file
+ FileType type = getFileType(fileName);
+ switch (type) {
+ case JOB_HISTORY_FILE:
+ if (jobFiles.getJobHistoryFilePath() == null) {
+ jobFiles.setJobHistoryFilePath(path);
+ } else {
+ LOG.warn("we already have the job history file " +
+ jobFiles.getJobHistoryFilePath() + ": skipping " + path);
+ }
+ break;
+ case JOB_CONF_FILE:
+ if (jobFiles.getJobConfFilePath() == null) {
+ jobFiles.setJobConfFilePath(path);
+ } else {
+ LOG.warn("we already have the job conf file " +
+ jobFiles.getJobConfFilePath() + ": skipping " + path);
+ }
+ break;
+ case UNKNOWN:
+ LOG.warn("unknown type: " + path);
+ }
+ }
+
+ private FileType getFileType(String fileName) {
+ if (fileName.endsWith(".jhist")) {
+ return FileType.JOB_HISTORY_FILE;
+ }
+ if (fileName.endsWith("_conf.xml")) {
+ return FileType.JOB_CONF_FILE;
+ }
+ return FileType.UNKNOWN;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
new file mode 100644
index 0000000..4ef0a14
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
@@ -0,0 +1,139 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.EntityWriter;
+import org.apache.hadoop.mapred.TimelineServicePerformanceV2.PerfCounters;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Adds simple entities with random string payload, events, metrics, and
+ * configuration.
+ */
+class SimpleEntityWriter extends EntityWriter {
+ private static final Log LOG = LogFactory.getLog(SimpleEntityWriter.class);
+
+ // constants for mtype = 1
+ static final String KBS_SENT = "kbs sent";
+ static final int KBS_SENT_DEFAULT = 1;
+ static final String TEST_TIMES = "testtimes";
+ static final int TEST_TIMES_DEFAULT = 100;
+ static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
+ "timeline.server.performance.run.id";
+
+ protected void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ // simulate the app id with the task id
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
+ ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
+
+ // create the app level timeline collector
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ manager.putIfAbsent(appId, collector);
+
+ try {
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(context.getJobName());
+ tlContext.setFlowRunId(timestamp);
+ tlContext.setUserId(context.getUser());
+
+ final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
+
+ long totalTime = 0;
+ final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
+ final Random rand = new Random();
+ final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
+ final char[] payLoad = new char[kbs * 1024];
+
+ for (int i = 0; i < testtimes; i++) {
+ // Generate a fixed length random payload
+ for (int xx = 0; xx < kbs * 1024; xx++) {
+ int alphaNumIdx =
+ rand.nextInt(TimelineServicePerformanceV2.alphaNums.length);
+ payLoad[xx] = TimelineServicePerformanceV2.alphaNums[alphaNumIdx];
+ }
+ String entId = taskAttemptId + "_" + Integer.toString(i);
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId(entId);
+ entity.setType("FOO_ATTEMPT");
+ entity.addInfo("PERF_TEST", payLoad);
+ // add an event
+ TimelineEvent event = new TimelineEvent();
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+ entity.addEvent(event);
+ // add a metric
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("foo_metric");
+ metric.addValue(System.currentTimeMillis(), 123456789L);
+ entity.addMetric(metric);
+ // add a config
+ entity.addConfig("foo", "bar");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ collector.putEntities(entities, ugi);
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ LOG.error("writing to the timeline service failed", e);
+ }
+ long endWrite = System.nanoTime();
+ totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
+ }
+ LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
+ " kB) in " + totalTime + " ms");
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(testtimes);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
+ increment(kbs*testtimes);
+ } finally {
+ // clean up
+ manager.remove(appId);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
new file mode 100644
index 0000000..80928dc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
@@ -0,0 +1,207 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapred;
+
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+class TimelineEntityConverter {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineEntityConverter.class);
+
+ static final String JOB = "MAPREDUCE_JOB";
+ static final String TASK = "MAPREDUCE_TASK";
+ static final String TASK_ATTEMPT = "MAPREDUCE_TASK_ATTEMPT";
+
+ /**
+ * Creates job, task, and task attempt entities based on the job history info
+ * and configuration.
+ *
+ * Note: currently these are plan timeline entities created for mapreduce
+ * types. These are not meant to be the complete and accurate entity set-up
+ * for mapreduce jobs. We do not leverage hierarchical timeline entities. If
+ * we create canonical mapreduce hierarchical timeline entities with proper
+ * parent-child relationship, we could modify this to use that instead.
+ *
+ * Note that we also do not add info to the YARN application entity, which
+ * would be needed for aggregation.
+ */
+ public Set<TimelineEntity> createTimelineEntities(JobInfo jobInfo,
+ Configuration conf) {
+ Set<TimelineEntity> entities = new HashSet<>();
+
+ // create the job entity
+ TimelineEntity job = createJobEntity(jobInfo, conf);
+ entities.add(job);
+
+ // create the task and task attempt entities
+ Set<TimelineEntity> tasksAndAttempts =
+ createTaskAndTaskAttemptEntities(jobInfo);
+ entities.addAll(tasksAndAttempts);
+
+ return entities;
+ }
+
+ private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
+ TimelineEntity job = new TimelineEntity();
+ job.setType(JOB);
+ job.setId(jobInfo.getJobId().toString());
+ job.setCreatedTime(jobInfo.getSubmitTime());
+
+ job.addInfo("JOBNAME", jobInfo.getJobname());
+ job.addInfo("USERNAME", jobInfo.getUsername());
+ job.addInfo("JOB_QUEUE_NAME", jobInfo.getJobQueueName());
+ job.addInfo("SUBMIT_TIME", jobInfo.getSubmitTime());
+ job.addInfo("LAUNCH_TIME", jobInfo.getLaunchTime());
+ job.addInfo("FINISH_TIME", jobInfo.getFinishTime());
+ job.addInfo("JOB_STATUS", jobInfo.getJobStatus());
+ job.addInfo("PRIORITY", jobInfo.getPriority());
+ job.addInfo("TOTAL_MAPS", jobInfo.getTotalMaps());
+ job.addInfo("TOTAL_REDUCES", jobInfo.getTotalReduces());
+ job.addInfo("UBERIZED", jobInfo.getUberized());
+ job.addInfo("ERROR_INFO", jobInfo.getErrorInfo());
+
+ // add metrics from total counters
+ // we omit the map counters and reduce counters for now as it's kind of
+ // awkward to put them (map/reduce/total counters are really a group of
+ // related counters)
+ Counters totalCounters = jobInfo.getTotalCounters();
+ if (totalCounters != null) {
+ addMetrics(job, totalCounters);
+ }
+ // finally add configuration to the job
+ addConfiguration(job, conf);
+ LOG.info("converted job " + jobInfo.getJobId() + " to a timeline entity");
+ return job;
+ }
+
+ private void addConfiguration(TimelineEntity job, Configuration conf) {
+ for (Map.Entry<String,String> e: conf) {
+ job.addConfig(e.getKey(), e.getValue());
+ }
+ }
+
+ private void addMetrics(TimelineEntity entity, Counters counters) {
+ for (CounterGroup g: counters) {
+ String groupName = g.getName();
+ for (Counter c: g) {
+ String name = groupName + ":" + c.getName();
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(name);
+ metric.addValue(System.currentTimeMillis(), c.getValue());
+ entity.addMetric(metric);
+ }
+ }
+ }
+
+ private Set<TimelineEntity> createTaskAndTaskAttemptEntities(JobInfo jobInfo) {
+ Set<TimelineEntity> entities = new HashSet<>();
+ Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
+ LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
+ " tasks");
+ for (TaskInfo taskInfo: taskInfoMap.values()) {
+ TimelineEntity task = createTaskEntity(taskInfo);
+ entities.add(task);
+ // add the task attempts from this task
+ Set<TimelineEntity> taskAttempts = createTaskAttemptEntities(taskInfo);
+ entities.addAll(taskAttempts);
+ }
+ return entities;
+ }
+
+ private TimelineEntity createTaskEntity(TaskInfo taskInfo) {
+ TimelineEntity task = new TimelineEntity();
+ task.setType(TASK);
+ task.setId(taskInfo.getTaskId().toString());
+ task.setCreatedTime(taskInfo.getStartTime());
+
+ task.addInfo("START_TIME", taskInfo.getStartTime());
+ task.addInfo("FINISH_TIME", taskInfo.getFinishTime());
+ task.addInfo("TASK_TYPE", taskInfo.getTaskType());
+ task.addInfo("TASK_STATUS", taskInfo.getTaskStatus());
+ task.addInfo("ERROR_INFO", taskInfo.getError());
+
+ // add metrics from counters
+ Counters counters = taskInfo.getCounters();
+ if (counters != null) {
+ addMetrics(task, counters);
+ }
+ LOG.info("converted task " + taskInfo.getTaskId() +
+ " to a timeline entity");
+ return task;
+ }
+
+ private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
+ Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
+ Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap =
+ taskInfo.getAllTaskAttempts();
+ LOG.info("task " + taskInfo.getTaskId() + " has " +
+ taskAttemptInfoMap.size() + " task attempts");
+ for (TaskAttemptInfo taskAttemptInfo: taskAttemptInfoMap.values()) {
+ TimelineEntity taskAttempt = createTaskAttemptEntity(taskAttemptInfo);
+ taskAttempts.add(taskAttempt);
+ }
+ return taskAttempts;
+ }
+
+ private TimelineEntity createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) {
+ TimelineEntity taskAttempt = new TimelineEntity();
+ taskAttempt.setType(TASK_ATTEMPT);
+ taskAttempt.setId(taskAttemptInfo.getAttemptId().toString());
+ taskAttempt.setCreatedTime(taskAttemptInfo.getStartTime());
+
+ taskAttempt.addInfo("START_TIME", taskAttemptInfo.getStartTime());
+ taskAttempt.addInfo("FINISH_TIME", taskAttemptInfo.getFinishTime());
+ taskAttempt.addInfo("MAP_FINISH_TIME",
+ taskAttemptInfo.getMapFinishTime());
+ taskAttempt.addInfo("SHUFFLE_FINISH_TIME",
+ taskAttemptInfo.getShuffleFinishTime());
+ taskAttempt.addInfo("SORT_FINISH_TIME",
+ taskAttemptInfo.getSortFinishTime());
+ taskAttempt.addInfo("TASK_STATUS", taskAttemptInfo.getTaskStatus());
+ taskAttempt.addInfo("STATE", taskAttemptInfo.getState());
+ taskAttempt.addInfo("ERROR", taskAttemptInfo.getError());
+ taskAttempt.addInfo("CONTAINER_ID",
+ taskAttemptInfo.getContainerId().toString());
+
+ // add metrics from counters
+ Counters counters = taskAttemptInfo.getCounters();
+ if (counters != null) {
+ addMetrics(taskAttempt, counters);
+ }
+ LOG.info("converted task attempt " + taskAttemptInfo.getAttemptId() +
+ " to a timeline entity");
+ return taskAttempt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
index 1c2e28d..f674ae1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
@@ -20,10 +20,7 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.util.Date;
-import java.util.Random;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.io.IntWritable;
@@ -31,49 +28,35 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.SleepJob.SleepInputFormat;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.GenericOptionsParser;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
public class TimelineServicePerformanceV2 extends Configured implements Tool {
- private static final Log LOG =
- LogFactory.getLog(TimelineServicePerformanceV2.class);
-
static final int NUM_MAPS_DEFAULT = 1;
static final int SIMPLE_ENTITY_WRITER = 1;
- // constants for mtype = 1
- static final String KBS_SENT = "kbs sent";
- static final int KBS_SENT_DEFAULT = 1;
- static final String TEST_TIMES = "testtimes";
- static final int TEST_TIMES_DEFAULT = 100;
- static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
- "timeline.server.performance.run.id";
-
+ static final int JOB_HISTORY_FILE_REPLAY_MAPPER = 2;
static int mapperType = SIMPLE_ENTITY_WRITER;
protected static int printUsage() {
- // TODO is there a way to handle mapper-specific options more gracefully?
System.err.println(
"Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
")\n" +
- " [-mtype <mapper type in integer>] \n" +
+ " [-mtype <mapper type in integer>]\n" +
" 1. simple entity write mapper\n" +
- " [-s <(KBs)test>] number of KB per put (default: " +
- KBS_SENT_DEFAULT + " KB)\n" +
- " [-t] package sending iterations per mapper (default: " +
- TEST_TIMES_DEFAULT + ")\n");
+ " 2. job history file replay mapper\n" +
+ " [-s <(KBs)test>] number of KB per put (mtype=1, default: " +
+ SimpleEntityWriter.KBS_SENT_DEFAULT + " KB)\n" +
+ " [-t] package sending iterations per mapper (mtype=1, default: " +
+ SimpleEntityWriter.TEST_TIMES_DEFAULT + ")\n" +
+ " [-d <path>] root path of job history files (mtype=2)\n" +
+ " [-r <replay mode>] (mtype=2)\n" +
+ " 1. write all entities for a job in one put (default)\n" +
+ " 2. write one entity at a time\n");
GenericOptionsParser.printGenericCommandUsage(System.err);
return -1;
}
@@ -82,11 +65,9 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
* Configure a job given argv.
*/
public static boolean parseArgs(String[] args, Job job) throws IOException {
- // set the defaults
+ // set the common defaults
Configuration conf = job.getConfiguration();
conf.setInt(MRJobConfig.NUM_MAPS, NUM_MAPS_DEFAULT);
- conf.setInt(KBS_SENT, KBS_SENT_DEFAULT);
- conf.setInt(TEST_TIMES, TEST_TIMES_DEFAULT);
for (int i = 0; i < args.length; i++) {
if (args.length == i + 1) {
@@ -97,25 +78,24 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
if ("-m".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
job.getConfiguration()
- .setInt(MRJobConfig.NUM_MAPS, (Integer.parseInt(args[i])));
+ .setInt(MRJobConfig.NUM_MAPS, Integer.parseInt(args[i]));
}
} else if ("-mtype".equals(args[i])) {
mapperType = Integer.parseInt(args[++i]);
- switch (mapperType) {
- case SIMPLE_ENTITY_WRITER:
- job.setMapperClass(SimpleEntityWriter.class);
- break;
- default:
- job.setMapperClass(SimpleEntityWriter.class);
- }
} else if ("-s".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(KBS_SENT, (Integer.parseInt(args[i])));
+ conf.setInt(SimpleEntityWriter.KBS_SENT, Integer.parseInt(args[i]));
}
} else if ("-t".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(TEST_TIMES, (Integer.parseInt(args[i])));
+ conf.setInt(SimpleEntityWriter.TEST_TIMES,
+ Integer.parseInt(args[i]));
}
+ } else if ("-d".equals(args[i])) {
+ conf.set(JobHistoryFileReplayMapper.PROCESSING_PATH, args[++i]);
+ } else if ("-r".equals(args[i])) {
+ conf.setInt(JobHistoryFileReplayMapper.REPLAY_MODE,
+ Integer.parseInt(args[++i]));
} else {
System.out.println("Unexpected argument: " + args[i]);
return printUsage() == 0;
@@ -128,6 +108,27 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
}
}
+ // handle mapper-specific settings
+ switch (mapperType) {
+ case JOB_HISTORY_FILE_REPLAY_MAPPER:
+ job.setMapperClass(JobHistoryFileReplayMapper.class);
+ String processingPath =
+ conf.get(JobHistoryFileReplayMapper.PROCESSING_PATH);
+ if (processingPath == null || processingPath.isEmpty()) {
+ System.out.println("processing path is missing while mtype = 2");
+ return printUsage() == 0;
+ }
+ break;
+ case SIMPLE_ENTITY_WRITER:
+ default:
+ job.setMapperClass(SimpleEntityWriter.class);
+ // use the current timestamp as the "run id" of the test: this will
+ // be used as simulating the cluster timestamp for apps
+ conf.setLong(SimpleEntityWriter.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
+ System.currentTimeMillis());
+ break;
+ }
+
return true;
}
@@ -153,13 +154,6 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
return -1;
}
- // for mtype = 1
- // use the current timestamp as the "run id" of the test: this will be used
- // as simulating the cluster timestamp for apps
- Configuration conf = job.getConfiguration();
- conf.setLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
- System.currentTimeMillis());
-
Date startTime = new Date();
System.out.println("Job started: " + startTime);
int ret = job.waitForCompletion(true) ? 0 : 1;
@@ -172,7 +166,8 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
double transacrate = writecounts * 1000 / (double)writetime;
double iorate = writesize * 1000 / (double)writetime;
- int numMaps = Integer.parseInt(conf.get(MRJobConfig.NUM_MAPS));
+ int numMaps =
+ Integer.parseInt(job.getConfiguration().get(MRJobConfig.NUM_MAPS));
System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
" ops/s");
@@ -204,95 +199,31 @@ public class TimelineServicePerformanceV2 extends Configured implements Tool {
'3', '4', '5', '6', '7', '8', '9', '0', ' ' };
/**
- * Adds simple entities with random string payload, events, metrics, and
- * configuration.
+ * Base mapper for writing entities to the timeline service. Subclasses
+ * override {@link #writeEntities(Configuration, TimelineCollectorManager,
+ * org.apache.hadoop.mapreduce.Mapper.Context)} to create and write entities
+ * to the timeline service.
*/
- public static class SimpleEntityWriter
+ public static abstract class EntityWriter
extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
+ @Override
public void map(IntWritable key, IntWritable val, Context context)
throws IOException {
- Configuration conf = context.getConfiguration();
- // simulate the app id with the task id
- int taskId = context.getTaskAttemptID().getTaskID().getId();
- long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
- ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
-
- // create the app level timeline collector
+ // create the timeline collector manager wired with the writer
Configuration tlConf = new YarnConfiguration();
- AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId);
- collector.init(tlConf);
- collector.start();
-
+ TimelineCollectorManager manager = new TimelineCollectorManager("test");
+ manager.init(tlConf);
+ manager.start();
try {
- // set the context
- // flow id: job name, flow run id: timestamp, user id
- TimelineCollectorContext tlContext =
- collector.getTimelineEntityContext();
- tlContext.setFlowName(context.getJobName());
- tlContext.setFlowRunId(timestamp);
- tlContext.setUserId(context.getUser());
-
- final int kbs = Integer.parseInt(conf.get(KBS_SENT));
-
- long totalTime = 0;
- final int testtimes = Integer.parseInt(conf.get(TEST_TIMES));
- final Random rand = new Random();
- final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
- final char[] payLoad = new char[kbs * 1024];
-
- for (int i = 0; i < testtimes; i++) {
- // Generate a fixed length random payload
- for (int xx = 0; xx < kbs * 1024; xx++) {
- int alphaNumIdx = rand.nextInt(alphaNums.length);
- payLoad[xx] = alphaNums[alphaNumIdx];
- }
- String entId = taskAttemptId + "_" + Integer.toString(i);
- final TimelineEntity entity = new TimelineEntity();
- entity.setId(entId);
- entity.setType("FOO_ATTEMPT");
- entity.addInfo("PERF_TEST", payLoad);
- // add an event
- TimelineEvent event = new TimelineEvent();
- event.setTimestamp(System.currentTimeMillis());
- event.addInfo("foo_event", "test");
- entity.addEvent(event);
- // add a metric
- TimelineMetric metric = new TimelineMetric();
- metric.setId("foo_metric");
- metric.addValue(System.currentTimeMillis(), 123456789L);
- entity.addMetric(metric);
- // add a config
- entity.addConfig("foo", "bar");
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
- // use the current user for this purpose
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- long startWrite = System.nanoTime();
- try {
- collector.putEntities(entities, ugi);
- } catch (Exception e) {
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
- increment(1);
- e.printStackTrace();
- }
- long endWrite = System.nanoTime();
- totalTime += (endWrite-startWrite)/1000000L;
- }
- LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
- " kB) in " + totalTime + " ms");
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
- increment(totalTime);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
- increment(testtimes);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
- increment(kbs*testtimes);
+ // invoke the method to have the subclass write entities
+ writeEntities(tlConf, manager, context);
} finally {
- // clean up
- collector.close();
+ manager.close();
}
}
+
+ protected abstract void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index e9f2085..165754d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -48,7 +48,7 @@ import com.google.common.annotations.VisibleForTesting;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public abstract class TimelineCollectorManager extends AbstractService {
+public class TimelineCollectorManager extends AbstractService {
private static final Log LOG =
LogFactory.getLog(TimelineCollectorManager.class);
@@ -90,10 +90,14 @@ public abstract class TimelineCollectorManager extends AbstractService {
Collections.synchronizedMap(
new HashMap<ApplicationId, TimelineCollector>());
- protected TimelineCollectorManager(String name) {
+ public TimelineCollectorManager(String name) {
super(name);
}
+ protected TimelineWriter getWriter() {
+ return writer;
+ }
+
/**
* Put the collector into the collection if an collector mapped by id does
* not exist.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index b22b39f..4385bbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -47,17 +47,17 @@ public class FileSystemTimelineWriterImpl extends AbstractService
private String outputRoot;
- /** Config param for timeline service storage tmp root for FILE YARN-3264 */
+ /** Config param for timeline service storage tmp root for FILE YARN-3264. */
public static final String TIMELINE_SERVICE_STORAGE_DIR_ROOT
- = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
+ = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "fs-writer.root-dir";
- /** default value for storage location on local disk */
+ /** default value for storage location on local disk. */
public static final String DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
- = "/tmp/timeline_service_data";
+ = "/tmp/timeline_service_data";
public static final String ENTITIES_DIR = "entities";
- /** Default extension for output files */
+ /** Default extension for output files. */
public static final String TIMELINE_SERVICE_STORAGE_EXTENSION = ".thist";
FileSystemTimelineWriterImpl() {
@@ -81,9 +81,11 @@ public class FileSystemTimelineWriterImpl extends AbstractService
TimelineWriteResponse response) throws IOException {
PrintWriter out = null;
try {
- String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,flowName,
- flowVersion, String.valueOf(flowRun), appId, entity.getType());
- String fileName = dir + entity.getId() + TIMELINE_SERVICE_STORAGE_EXTENSION;
+ String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,
+ escape(flowName), escape(flowVersion), String.valueOf(flowRun), appId,
+ entity.getType());
+ String fileName = dir + entity.getId() +
+ TIMELINE_SERVICE_STORAGE_EXTENSION;
out =
new PrintWriter(new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(fileName, true), "UTF-8")));
@@ -145,4 +147,9 @@ public class FileSystemTimelineWriterImpl extends AbstractService
}
return path.toString();
}
+
+ // specifically escape the separator character
+ private static String escape(String str) {
+ return str.replace(File.separatorChar, '_');
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09a8b7b9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
new file mode 100644
index 0000000..f652ffd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
[44/50] [abbrv] hadoop git commit: MAPREDUCE-6370. Made the timeline
service v2 test driver write event ID. Contributed by Li Lu.
Posted by sj...@apache.org.
MAPREDUCE-6370. Made the timeline service v2 test driver write event ID. Contributed by Li Lu.
(cherry picked from commit 827633ee9fee26e3e15343cbccb0b7905ae02170)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/40d9d469
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/40d9d469
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/40d9d469
Branch: refs/heads/YARN-2928
Commit: 40d9d4690568476fb43c006983a91304434aa066
Parents: 09a8b7b
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 22 00:00:05 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:16 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40d9d469/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 9c66a5e..e6348b9 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -21,6 +21,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
BUG FIXES
+ MAPREDUCE-6370. Made the timeline service v2 test driver write event ID.
+ (Li Lu via zjshen)
+
Trunk (Unreleased)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/40d9d469/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
index 4ef0a14..625c32a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
@@ -97,6 +97,7 @@ class SimpleEntityWriter extends EntityWriter {
entity.addInfo("PERF_TEST", payLoad);
// add an event
TimelineEvent event = new TimelineEvent();
+ event.setId("foo_event_id");
event.setTimestamp(System.currentTimeMillis());
event.addInfo("foo_event", "test");
entity.addEvent(event);
[50/50] [abbrv] hadoop git commit: YARN-3058. Miscellaneous issues in
NodeManager project (Naganarasimha G R via sjlee)
Posted by sj...@apache.org.
YARN-3058. Miscellaneous issues in NodeManager project (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/be95107a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/be95107a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/be95107a
Branch: refs/heads/YARN-2928
Commit: be95107aa5fc4de6c8c3f10e55a1495d305d6ab4
Parents: afa54d2
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Aug 24 17:36:31 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:52:45 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../containermanager/ContainerManagerImpl.java | 33 +++++++++++---------
.../metrics/TimelineServiceV2Publisher.java | 2 --
.../TestSystemMetricsPublisherForV2.java | 8 ++++-
4 files changed, 28 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be95107a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b1d2bd6..4414178 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -144,6 +144,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-4064. build is broken at TestHBaseTimelineWriterImpl.java (sjlee)
+ YARN-3058. Miscellaneous issues in NodeManager project (Naganarasimha G R
+ via sjlee)
+
Trunk - Unreleased
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be95107a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 2ea2ec1..50f2dfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -896,21 +896,24 @@ public class ContainerManagerImpl extends CompositeService implements
if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
flowRunId = Long.parseLong(flowRunIdStr);
}
- Application application = new ApplicationImpl(dispatcher, user,
- flowName, flowVersion, flowRunId, applicationID, credentials, context);
- if (null == context.getApplications().putIfAbsent(applicationID,
- application)) {
- LOG.info("Creating a new application reference for app " + applicationID);
- LogAggregationContext logAggregationContext =
- containerTokenIdentifier.getLogAggregationContext();
- Map<ApplicationAccessType, String> appAcls =
- container.getLaunchContext().getApplicationACLs();
- context.getNMStateStore().storeApplication(applicationID,
- buildAppProto(applicationID, user, credentials, appAcls,
- logAggregationContext));
- dispatcher.getEventHandler().handle(
- new ApplicationInitEvent(applicationID, appAcls,
- logAggregationContext));
+ if (!context.getApplications().containsKey(applicationID)) {
+ Application application =
+ new ApplicationImpl(dispatcher, user, flowName, flowVersion,
+ flowRunId, applicationID, credentials, context);
+ if (context.getApplications().putIfAbsent(applicationID,
+ application) == null) {
+ LOG.info("Creating a new application reference for app "
+ + applicationID);
+ LogAggregationContext logAggregationContext =
+ containerTokenIdentifier.getLogAggregationContext();
+ Map<ApplicationAccessType, String> appAcls =
+ container.getLaunchContext().getApplicationACLs();
+ context.getNMStateStore().storeApplication(applicationID,
+ buildAppProto(applicationID, user, credentials, appAcls,
+ logAggregationContext));
+ dispatcher.getEventHandler().handle(new ApplicationInitEvent(
+ applicationID, appAcls, logAggregationContext));
+ }
}
dispatcher.getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be95107a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 700708a..e8677e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -39,8 +39,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identif
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
-import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/be95107a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 9830a80..ac20335 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
@@ -45,7 +46,6 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -332,6 +332,12 @@ public class TestSystemMetricsPublisherForV2 {
when(app.getRMAppMetrics()).thenReturn(
new RMAppMetrics(null, 0, 0, Integer.MAX_VALUE, Long.MAX_VALUE));
when(app.getApplicationTags()).thenReturn(Collections.<String> emptySet());
+ ApplicationSubmissionContext appSubmissionContext =
+ mock(ApplicationSubmissionContext.class);
+ when(appSubmissionContext.getPriority())
+ .thenReturn(Priority.newInstance(0));
+ when(app.getApplicationSubmissionContext())
+ .thenReturn(appSubmissionContext);
return app;
}
[43/50] [abbrv] hadoop git commit: YARN-3904. Refactor
timelineservice.storage to add support to online and offline aggregation
writers (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-3904. Refactor timelineservice.storage to add support to online and offline aggregation writers (Li Lu via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97f211b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97f211b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97f211b4
Branch: refs/heads/YARN-2928
Commit: 97f211b485a4fb5f49ce8775d873afba869f8e7c
Parents: 40d9d46
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Aug 17 16:48:58 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:16 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 7 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 10 +
.../storage/OfflineAggregationWriter.java | 66 +++
.../PhoenixOfflineAggregationWriterImpl.java | 356 +++++++++++++
.../storage/PhoenixTimelineWriterImpl.java | 530 -------------------
.../storage/TimelineSchemaCreator.java | 45 +-
.../storage/common/OfflineAggregationInfo.java | 110 ++++
...TestPhoenixOfflineAggregationWriterImpl.java | 162 ++++++
.../storage/TestPhoenixTimelineWriterImpl.java | 152 ------
.../storage/TestTimelineWriterImpl.java | 74 ---
11 files changed, 754 insertions(+), 761 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a4aeef8..3e1f212 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -88,6 +88,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3906. Split the application table from the entity table. (Sangjin Lee
via junping_du)
+ YARN-3904. Refactor timelineservice.storage to add support to online and
+ offline aggregation writers (Li Lu via sjlee)
+
IMPROVEMENTS
YARN-3276. Code cleanup for timeline service API records. (Junping Du via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 5583cd6..691170e 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -488,13 +488,12 @@
<!-- Ignore SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING warnings for Timeline Phoenix storage. -->
<!-- Since we're using dynamic columns, we have to generate SQL statements dynamically -->
<Match>
- <Class name="org.apache.hadoop.yarn.server.timelineservice.storage.PhoenixTimelineWriterImpl" />
+ <Class name="org.apache.hadoop.yarn.server.timelineservice.storage.PhoenixOfflineAggregationWriterImpl" />
<Or>
<Method name="storeEntityVariableLengthFields" />
- <Method name="storeEvents" />
- <Method name="storeMetrics" />
- <Method name="write" />
+ <Method name="writeAggregatedEntity" />
</Or>
+ <Bug pattern="SQL_PREPARED_STATEMENT_GENERATED_FROM_NONCONSTANT_STRING" />
</Match>
<!-- Following fields are used in ErrorsAndWarningsBlock, which is not a part of analysis of findbugs -->
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2c59bd4..faab8a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1737,6 +1737,16 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_TIMELINE_DELEGATION_TOKEN_MAX_LIFETIME =
7*24*60*60*1000; // 7 days
+ // Timeline service v2 offlien aggregation related keys
+ public static final String TIMELINE_OFFLINE_AGGREGATION_PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + "aggregation.offline.";
+ public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR
+ = TIMELINE_OFFLINE_AGGREGATION_PREFIX
+ + "phoenix.connectionString";
+
+ public static final String PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT
+ = "jdbc:phoenix:localhost:2181:/hbase";
+
// ///////////////////////////////
// Shared Cache Configs
// ///////////////////////////////
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
new file mode 100644
index 0000000..e1219e0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggregationInfo;
+
+import java.io.IOException;
+
+/**
+ * YARN timeline service v2 offline aggregation storage interface
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public abstract class OfflineAggregationWriter extends AbstractService {
+
+ /**
+ * Construct the offline writer.
+ *
+ * @param name service name
+ */
+ public OfflineAggregationWriter(String name) {
+ super(name);
+ }
+
+ /**
+ * Persist aggregated timeline entities to the offline store based on which
+ * track this entity is to be rolled up to. The tracks along which aggregations
+ * are to be done are given by {@link OfflineAggregationInfo}.
+ *
+ * @param context a {@link TimelineCollectorContext} object that describes the
+ * context information of the aggregated data. Depends on the
+ * type of the aggregation, some fields of this context maybe
+ * empty or null.
+ * @param entities {@link TimelineEntities} to be persisted.
+ * @param info an {@link OfflineAggregationInfo} object that describes the
+ * detail of the aggregation. Current supported option is
+ * {@link OfflineAggregationInfo#FLOW_AGGREGATION}.
+ * @return a {@link TimelineWriteResponse} object.
+ * @throws IOException
+ */
+ abstract TimelineWriteResponse writeAggregatedEntity(
+ TimelineCollectorContext context,
+ TimelineEntities entities, OfflineAggregationInfo info) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
new file mode 100644
index 0000000..4c1352c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
@@ -0,0 +1,356 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggregationInfo;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.phoenix.util.PropertiesUtil;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+/**
+ * Offline aggregation Phoenix storage. This storage currently consists of two
+ * aggregation tables, one for flow level aggregation and one for user level
+ * aggregation.
+ *
+ * Example table record:
+ *
+ * <pre>
+ * |---------------------------|
+ * | Primary | Column Family|
+ * | key | metrics |
+ * |---------------------------|
+ * | row_key | metricId1: |
+ * | | metricValue1 |
+ * | | @timestamp1 |
+ * | | |
+ * | | metriciD1: |
+ * | | metricValue2 |
+ * | | @timestamp2 |
+ * | | |
+ * | | metricId2: |
+ * | | metricValue1 |
+ * | | @timestamp2 |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * | | |
+ * |---------------------------|
+ * </pre>
+ *
+ * For the flow aggregation table, the primary key contains user, cluster, and
+ * flow id. For user aggregation table,the primary key is user.
+ *
+ * Metrics column family stores all aggregated metrics for each record.
+ */
+@Private
+@Unstable
+public class PhoenixOfflineAggregationWriterImpl
+ extends OfflineAggregationWriter {
+
+ private static final Log LOG
+ = LogFactory.getLog(PhoenixOfflineAggregationWriterImpl.class);
+ private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
+ = "timeline_cf_placeholder";
+
+ /** Default Phoenix JDBC driver name */
+ private static final String DRIVER_CLASS_NAME
+ = "org.apache.phoenix.jdbc.PhoenixDriver";
+
+ /** Default Phoenix timeline config column family */
+ private static final String METRIC_COLUMN_FAMILY = "m.";
+ /** Default Phoenix timeline info column family */
+ private static final String INFO_COLUMN_FAMILY = "i.";
+ /** Default separator for Phoenix storage */
+ private static final String AGGREGATION_STORAGE_SEPARATOR = ";";
+
+ /** Connection string to the deployed Phoenix cluster */
+ private String connString = null;
+ private Properties connProperties = new Properties();
+
+ public PhoenixOfflineAggregationWriterImpl(Properties prop) {
+ super(PhoenixOfflineAggregationWriterImpl.class.getName());
+ connProperties = PropertiesUtil.deepCopy(prop);
+ }
+
+ public PhoenixOfflineAggregationWriterImpl() {
+ super(PhoenixOfflineAggregationWriterImpl.class.getName());
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ Class.forName(DRIVER_CLASS_NAME);
+ // so check it here and only read in the config if it's not overridden.
+ connString =
+ conf.get(YarnConfiguration.PHOENIX_OFFLINE_STORAGE_CONN_STR,
+ YarnConfiguration.PHOENIX_OFFLINE_STORAGE_CONN_STR_DEFAULT);
+ super.init(conf);
+ }
+
+ @Override
+ public TimelineWriteResponse writeAggregatedEntity(
+ TimelineCollectorContext context, TimelineEntities entities,
+ OfflineAggregationInfo info) throws IOException {
+ TimelineWriteResponse response = new TimelineWriteResponse();
+ String sql = "UPSERT INTO " + info.getTableName()
+ + " (" + StringUtils.join(info.getPrimaryKeyList(), ",")
+ + ", created_time, modified_time, metric_names) "
+ + "VALUES ("
+ + StringUtils.repeat("?,", info.getPrimaryKeyList().length)
+ + "?, ?, ?)";
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("TimelineEntity write SQL: " + sql);
+ }
+
+ try (Connection conn = getConnection();
+ PreparedStatement ps = conn.prepareStatement(sql)) {
+ for (TimelineEntity entity : entities.getEntities()) {
+ HashMap<String, TimelineMetric> formattedMetrics = new HashMap<>();
+ if (entity.getMetrics() != null) {
+ for (TimelineMetric m : entity.getMetrics()) {
+ formattedMetrics.put(m.getId(), m);
+ }
+ }
+ int idx = info.setStringsForPrimaryKey(ps, context, null, 1);
+ ps.setLong(idx++, entity.getCreatedTime());
+ ps.setLong(idx++, entity.getModifiedTime());
+ ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(),
+ AGGREGATION_STORAGE_SEPARATOR));
+ ps.execute();
+
+ storeEntityVariableLengthFields(entity, formattedMetrics, context, conn,
+ info);
+
+ conn.commit();
+ }
+ } catch (SQLException se) {
+ LOG.error("Failed to add entity to Phoenix " + se.getMessage());
+ throw new IOException(se);
+ } catch (Exception e) {
+ LOG.error("Exception on getting connection: " + e.getMessage());
+ throw new IOException(e);
+ }
+ return response;
+ }
+
+ /**
+ * Create Phoenix tables for offline aggregation storage if the tables do not
+ * exist.
+ *
+ * @throws IOException
+ */
+ public void createPhoenixTables() throws IOException {
+ // Create tables if necessary
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ // Table schema defined as in YARN-3817.
+ String sql = "CREATE TABLE IF NOT EXISTS "
+ + OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "flow_name VARCHAR NOT NULL, "
+ + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + "metric_names VARCHAR, info_keys VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY("
+ + "user, cluster, flow_name))";
+ stmt.executeUpdate(sql);
+ sql = "CREATE TABLE IF NOT EXISTS "
+ + OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME
+ + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + "metric_names VARCHAR, info_keys VARCHAR "
+ + "CONSTRAINT pk PRIMARY KEY(user, cluster))";
+ stmt.executeUpdate(sql);
+ conn.commit();
+ } catch (SQLException se) {
+ LOG.error("Failed in init data " + se.getLocalizedMessage());
+ throw new IOException(se);
+ }
+ return;
+ }
+
+ // Utility functions
+ @Private
+ @VisibleForTesting
+ Connection getConnection() throws IOException {
+ Connection conn;
+ try {
+ conn = DriverManager.getConnection(connString, connProperties);
+ conn.setAutoCommit(false);
+ } catch (SQLException se) {
+ LOG.error("Failed to connect to phoenix server! "
+ + se.getLocalizedMessage());
+ throw new IOException(se);
+ }
+ return conn;
+ }
+
+ // WARNING: This method will permanently drop a table!
+ @Private
+ @VisibleForTesting
+ void dropTable(String tableName) throws Exception {
+ try (Connection conn = getConnection();
+ Statement stmt = conn.createStatement()) {
+ String sql = "DROP TABLE " + tableName;
+ stmt.executeUpdate(sql);
+ } catch (SQLException se) {
+ LOG.error("Failed in dropping entity table " + se.getLocalizedMessage());
+ throw se;
+ }
+ }
+
+ private static class DynamicColumns<K> {
+ static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
+ static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
+ String columnFamilyPrefix;
+ String type;
+ Set<K> columns;
+
+ public DynamicColumns(String columnFamilyPrefix, String type,
+ Set<K> keyValues) {
+ this.columnFamilyPrefix = columnFamilyPrefix;
+ this.columns = keyValues;
+ this.type = type;
+ }
+ }
+
+ private static <K> StringBuilder appendColumnsSQL(
+ StringBuilder colNames, DynamicColumns<K> cfInfo) {
+ // Prepare the sql template by iterating through all keys
+ for (K key : cfInfo.columns) {
+ colNames.append(",").append(cfInfo.columnFamilyPrefix)
+ .append(key.toString()).append(cfInfo.type);
+ }
+ return colNames;
+ }
+
+ private static <K, V> int setValuesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos,
+ boolean converToBytes) throws SQLException {
+ int idx = startPos;
+ for (Map.Entry<K, V> entry : keyValues.entrySet()) {
+ V value = entry.getValue();
+ if (value instanceof Collection) {
+ ps.setString(idx++, StringUtils.join(
+ (Collection) value, AGGREGATION_STORAGE_SEPARATOR));
+ } else {
+ if (converToBytes) {
+ try {
+ ps.setBytes(idx++, GenericObjectMapper.write(entry.getValue()));
+ } catch (IOException ie) {
+ LOG.error("Exception in converting values into bytes "
+ + ie.getMessage());
+ throw new SQLException(ie);
+ }
+ } else {
+ ps.setString(idx++, value.toString());
+ }
+ }
+ }
+ return idx;
+ }
+
+ private static <K, V> int setBytesForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, true);
+ }
+
+ private static <K, V> int setStringsForColumnFamily(
+ PreparedStatement ps, Map<K, V> keyValues, int startPos)
+ throws SQLException {
+ return setValuesForColumnFamily(ps, keyValues, startPos, false);
+ }
+
+ private static void storeEntityVariableLengthFields(TimelineEntity entity,
+ Map<String, TimelineMetric> formattedMetrics,
+ TimelineCollectorContext context, Connection conn,
+ OfflineAggregationInfo aggregationInfo) throws SQLException {
+ int numPlaceholders = 0;
+ StringBuilder columnDefs = new StringBuilder(
+ StringUtils.join(aggregationInfo.getPrimaryKeyList(), ","));
+ if (formattedMetrics != null && formattedMetrics.size() > 0) {
+ appendColumnsSQL(columnDefs, new DynamicColumns<>(
+ METRIC_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
+ formattedMetrics.keySet()));
+ numPlaceholders += formattedMetrics.keySet().size();
+ }
+ if (numPlaceholders == 0) {
+ return;
+ }
+ StringBuilder placeholders = new StringBuilder();
+ placeholders.append(
+ StringUtils.repeat("?,", aggregationInfo.getPrimaryKeyList().length));
+ // numPlaceholders >= 1 now
+ placeholders.append("?")
+ .append(StringUtils.repeat(",?", numPlaceholders - 1));
+ String sqlVariableLengthFields = new StringBuilder("UPSERT INTO ")
+ .append(aggregationInfo.getTableName()).append(" (").append(columnDefs)
+ .append(") VALUES(").append(placeholders).append(")").toString();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("SQL statement for variable length fields: "
+ + sqlVariableLengthFields);
+ }
+ // Use try with resource statement for the prepared statement
+ try (PreparedStatement psVariableLengthFields =
+ conn.prepareStatement(sqlVariableLengthFields)) {
+ int idx = aggregationInfo.setStringsForPrimaryKey(
+ psVariableLengthFields, context, null, 1);
+ if (formattedMetrics != null && formattedMetrics.size() > 0) {
+ idx = setBytesForColumnFamily(
+ psVariableLengthFields, formattedMetrics, idx);
+ }
+ psVariableLengthFields.execute();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
deleted file mode 100644
index 381ff17..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixTimelineWriterImpl.java
+++ /dev/null
@@ -1,530 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-
-import java.io.IOException;
-import java.sql.Connection;
-import java.sql.DriverManager;
-import java.sql.PreparedStatement;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.Map;
-import java.util.Properties;
-import java.util.Set;
-
-@Private
-@Unstable
-public class PhoenixTimelineWriterImpl extends AbstractService
- implements TimelineWriter {
-
- public static final String TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR
- = YarnConfiguration.TIMELINE_SERVICE_PREFIX
- + "writer.phoenix.connectionString";
-
- public static final String TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT
- = "jdbc:phoenix:localhost:2181:/hbase";
-
- private static final Log LOG
- = LogFactory.getLog(PhoenixTimelineWriterImpl.class);
- private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
- = "timeline_cf_placeholder";
- // These lists are not taking effects in table creations.
- private static final String[] PHOENIX_STORAGE_PK_LIST
- = {"cluster", "user", "flow_name", "flow_version", "flow_run", "app_id",
- "type", "entity_id"};
- private static final String[] TIMELINE_EVENT_EXTRA_PK_LIST =
- {"timestamp", "event_id"};
- private static final String[] TIMELINE_METRIC_EXTRA_PK_LIST =
- {"metric_id"};
- /** Default Phoenix JDBC driver name */
- private static final String DRIVER_CLASS_NAME
- = "org.apache.phoenix.jdbc.PhoenixDriver";
-
- /** Default Phoenix timeline entity table name */
- @VisibleForTesting
- static final String ENTITY_TABLE_NAME = "timeline_entity";
- /** Default Phoenix event table name */
- @VisibleForTesting
- static final String EVENT_TABLE_NAME = "timeline_event";
- /** Default Phoenix metric table name */
- @VisibleForTesting
- static final String METRIC_TABLE_NAME = "metric_singledata";
-
- /** Default Phoenix timeline config column family */
- private static final String CONFIG_COLUMN_FAMILY = "c.";
- /** Default Phoenix timeline info column family */
- private static final String INFO_COLUMN_FAMILY = "i.";
- /** Default Phoenix event info column family */
- private static final String EVENT_INFO_COLUMN_FAMILY = "ei.";
- /** Default Phoenix isRelatedTo column family */
- private static final String IS_RELATED_TO_FAMILY = "ir.";
- /** Default Phoenix relatesTo column family */
- private static final String RELATES_TO_FAMILY = "rt.";
- /** Default separator for Phoenix storage */
- private static final String PHOENIX_STORAGE_SEPARATOR = ";";
-
- /** Connection string to the deployed Phoenix cluster */
- @VisibleForTesting
- String connString = null;
- @VisibleForTesting
- Properties connProperties = new Properties();
-
- PhoenixTimelineWriterImpl() {
- super((PhoenixTimelineWriterImpl.class.getName()));
- }
-
- @Override
- protected void serviceInit(Configuration conf) throws Exception {
- // so check it here and only read in the config if it's not overridden.
- connString =
- conf.get(TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
- TIMELINE_SERVICE_PHEONIX_STORAGE_CONN_STR_DEFAULT);
- createTables();
- super.init(conf);
- }
-
- @Override
- protected void serviceStop() throws Exception {
- super.serviceStop();
- }
-
- @Override
- public TimelineWriteResponse write(String clusterId, String userId,
- String flowName, String flowVersion, long flowRunId, String appId,
- TimelineEntities entities) throws IOException {
- TimelineWriteResponse response = new TimelineWriteResponse();
- TimelineCollectorContext currContext = new TimelineCollectorContext(
- clusterId, userId, flowName, flowVersion, flowRunId, appId);
- String sql = "UPSERT INTO " + ENTITY_TABLE_NAME
- + " (" + StringUtils.join(PHOENIX_STORAGE_PK_LIST, ",")
- + ", creation_time, modified_time, configs) "
- + "VALUES (" + StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length)
- + "?, ?, ?)";
- if (LOG.isDebugEnabled()) {
- LOG.debug("TimelineEntity write SQL: " + sql);
- }
-
- try (Connection conn = getConnection();
- PreparedStatement ps = conn.prepareStatement(sql)) {
- for (TimelineEntity entity : entities.getEntities()) {
- int idx = setStringsForPrimaryKey(ps, currContext, entity, 1);
- ps.setLong(idx++, entity.getCreatedTime());
- ps.setLong(idx++, entity.getModifiedTime());
- String configKeys = StringUtils.join(
- entity.getConfigs().keySet(), PHOENIX_STORAGE_SEPARATOR);
- ps.setString(idx++, configKeys);
- ps.execute();
-
- storeEntityVariableLengthFields(entity, currContext, conn);
- storeEvents(entity, currContext, conn);
- storeMetrics(entity, currContext, conn);
-
- conn.commit();
- }
- } catch (SQLException se) {
- LOG.error("Failed to add entity to Phoenix " + se.getMessage());
- throw new IOException(se);
- } catch (Exception e) {
- LOG.error("Exception on getting connection: " + e.getMessage());
- throw new IOException(e);
- }
- return response;
- }
-
- /**
- * Aggregates the entity information to the timeline store based on which
- * track this entity is to be rolled up to The tracks along which aggregations
- * are to be done are given by {@link TimelineAggregationTrack}
- *
- * Any errors occurring for individual write request objects will be reported
- * in the response.
- *
- * @param data
- * a {@link TimelineEntity} object
- * a {@link TimelineAggregationTrack} enum value
- * @return a {@link TimelineWriteResponse} object.
- * @throws IOException
- */
- @Override
- public TimelineWriteResponse aggregate(TimelineEntity data,
- TimelineAggregationTrack track) throws IOException {
- return null;
-
- }
-
- @Override
- public void flush() throws IOException {
- // currently no-op
- }
-
- // Utility functions
- @Private
- @VisibleForTesting
- Connection getConnection() throws IOException {
- Connection conn;
- try {
- Class.forName(DRIVER_CLASS_NAME);
- conn = DriverManager.getConnection(connString, connProperties);
- conn.setAutoCommit(false);
- } catch (SQLException se) {
- LOG.error("Failed to connect to phoenix server! "
- + se.getLocalizedMessage());
- throw new IOException(se);
- } catch (ClassNotFoundException e) {
- LOG.error("Class not found! " + e.getLocalizedMessage());
- throw new IOException(e);
- }
- return conn;
- }
-
- private void createTables() throws Exception {
- // Create tables if necessary
- try (Connection conn = getConnection();
- Statement stmt = conn.createStatement()) {
- // Table schema defined as in YARN-3134.
- String sql = "CREATE TABLE IF NOT EXISTS " + ENTITY_TABLE_NAME
- + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
- + "flow_run UNSIGNED_LONG NOT NULL, "
- + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
- + "entity_id VARCHAR NOT NULL, "
- + "creation_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
- + "configs VARCHAR, "
- + CONFIG_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
- + INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
- + IS_RELATED_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR, "
- + RELATES_TO_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARCHAR "
- + "CONSTRAINT pk PRIMARY KEY("
- + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
- + "type, entity_id))";
- stmt.executeUpdate(sql);
- sql = "CREATE TABLE IF NOT EXISTS " + EVENT_TABLE_NAME
- + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
- + "flow_run UNSIGNED_LONG NOT NULL, "
- + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
- + "entity_id VARCHAR NOT NULL, "
- + "timestamp UNSIGNED_LONG NOT NULL, event_id VARCHAR NOT NULL, "
- + EVENT_INFO_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY "
- + "CONSTRAINT pk PRIMARY KEY("
- + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
- + "type, entity_id, timestamp DESC, event_id))";
- stmt.executeUpdate(sql);
- sql = "CREATE TABLE IF NOT EXISTS " + METRIC_TABLE_NAME
- + "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "flow_name VARCHAR NOT NULL, flow_version VARCHAR NOT NULL, "
- + "flow_run UNSIGNED_LONG NOT NULL, "
- + "app_id VARCHAR NOT NULL, type VARCHAR NOT NULL, "
- + "entity_id VARCHAR NOT NULL, "
- + "metric_id VARCHAR NOT NULL, "
- + "singledata VARBINARY, "
- + "time UNSIGNED_LONG "
- + "CONSTRAINT pk PRIMARY KEY("
- + "user, cluster, flow_name, flow_version, flow_run DESC, app_id, "
- + "type, entity_id, metric_id))";
- stmt.executeUpdate(sql);
- conn.commit();
- } catch (SQLException se) {
- LOG.error("Failed in init data " + se.getLocalizedMessage());
- throw se;
- }
- return;
- }
-
- private static class DynamicColumns<K> {
- static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
- static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
- String columnFamilyPrefix;
- String type;
- Set<K> columns;
-
- public DynamicColumns(String columnFamilyPrefix, String type,
- Set<K> keyValues) {
- this.columnFamilyPrefix = columnFamilyPrefix;
- this.columns = keyValues;
- this.type = type;
- }
- }
-
- private static <K> StringBuilder appendColumnsSQL(
- StringBuilder colNames, DynamicColumns<K> cfInfo) {
- // Prepare the sql template by iterating through all keys
- for (K key : cfInfo.columns) {
- colNames.append(",").append(cfInfo.columnFamilyPrefix)
- .append(key.toString()).append(cfInfo.type);
- }
- return colNames;
- }
-
- private static <K, V> int setValuesForColumnFamily(
- PreparedStatement ps, Map<K, V> keyValues, int startPos,
- boolean converToBytes) throws SQLException {
- int idx = startPos;
- for (Map.Entry<K, V> entry : keyValues.entrySet()) {
- V value = entry.getValue();
- if (value instanceof Collection) {
- ps.setString(idx++, StringUtils.join(
- (Collection) value, PHOENIX_STORAGE_SEPARATOR));
- } else {
- if (converToBytes) {
- try {
- ps.setBytes(idx++, GenericObjectMapper.write(entry.getValue()));
- } catch (IOException ie) {
- LOG.error("Exception in converting values into bytes "
- + ie.getMessage());
- throw new SQLException(ie);
- }
- } else {
- ps.setString(idx++, value.toString());
- }
- }
- }
- return idx;
- }
-
- private static <K, V> int setBytesForColumnFamily(
- PreparedStatement ps, Map<K, V> keyValues, int startPos)
- throws SQLException {
- return setValuesForColumnFamily(ps, keyValues, startPos, true);
- }
-
- private static <K, V> int setStringsForColumnFamily(
- PreparedStatement ps, Map<K, V> keyValues, int startPos)
- throws SQLException {
- return setValuesForColumnFamily(ps, keyValues, startPos, false);
- }
-
- private static int setStringsForPrimaryKey(PreparedStatement ps,
- TimelineCollectorContext context, TimelineEntity entity, int startPos)
- throws SQLException {
- int idx = startPos;
- ps.setString(idx++, context.getClusterId());
- ps.setString(idx++, context.getUserId());
- ps.setString(idx++,
- context.getFlowName());
- ps.setString(idx++, context.getFlowVersion());
- ps.setLong(idx++, context.getFlowRunId());
- ps.setString(idx++, context.getAppId());
- ps.setString(idx++, entity.getType());
- ps.setString(idx++, entity.getId());
- return idx;
- }
-
- private static void storeEntityVariableLengthFields(TimelineEntity entity,
- TimelineCollectorContext context, Connection conn) throws SQLException {
- int numPlaceholders = 0;
- StringBuilder columnDefs = new StringBuilder(
- StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
- if (entity.getConfigs() != null) {
- Set<String> keySet = entity.getConfigs().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- CONFIG_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (entity.getInfo() != null) {
- Set<String> keySet = entity.getInfo().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (entity.getIsRelatedToEntities() != null) {
- Set<String> keySet = entity.getIsRelatedToEntities().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- IS_RELATED_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (entity.getRelatesToEntities() != null) {
- Set<String> keySet = entity.getRelatesToEntities().keySet();
- appendColumnsSQL(columnDefs, new DynamicColumns<>(
- RELATES_TO_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_STRING,
- keySet));
- numPlaceholders += keySet.size();
- }
- if (numPlaceholders == 0) {
- return;
- }
- StringBuilder placeholders = new StringBuilder();
- placeholders.append(
- StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length));
- // numPlaceholders >= 1 now
- placeholders.append("?")
- .append(StringUtils.repeat(",?", numPlaceholders - 1));
- String sqlVariableLengthFields = new StringBuilder("UPSERT INTO ")
- .append(ENTITY_TABLE_NAME).append(" (").append(columnDefs)
- .append(") VALUES(").append(placeholders).append(")").toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL statement for variable length fields: "
- + sqlVariableLengthFields);
- }
- // Use try with resource statement for the prepared statement
- try (PreparedStatement psVariableLengthFields =
- conn.prepareStatement(sqlVariableLengthFields)) {
- int idx = setStringsForPrimaryKey(
- psVariableLengthFields, context, entity, 1);
- if (entity.getConfigs() != null) {
- idx = setStringsForColumnFamily(
- psVariableLengthFields, entity.getConfigs(), idx);
- }
- if (entity.getInfo() != null) {
- idx = setBytesForColumnFamily(
- psVariableLengthFields, entity.getInfo(), idx);
- }
- if (entity.getIsRelatedToEntities() != null) {
- idx = setStringsForColumnFamily(
- psVariableLengthFields, entity.getIsRelatedToEntities(), idx);
- }
- if (entity.getRelatesToEntities() != null) {
- idx = setStringsForColumnFamily(
- psVariableLengthFields, entity.getRelatesToEntities(), idx);
- }
- psVariableLengthFields.execute();
- }
- }
-
- private static void storeMetrics(TimelineEntity entity,
- TimelineCollectorContext context, Connection conn) throws SQLException {
- if (entity.getMetrics() == null) {
- return;
- }
- Set<TimelineMetric> metrics = entity.getMetrics();
- for (TimelineMetric metric : metrics) {
- StringBuilder sqlColumns = new StringBuilder(
- StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
- sqlColumns.append(",")
- .append(StringUtils.join(TIMELINE_METRIC_EXTRA_PK_LIST, ","));
- sqlColumns.append(",").append("singledata, time");
- StringBuilder placeholders = new StringBuilder();
- placeholders.append(
- StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
- .append(StringUtils.repeat("?,", TIMELINE_METRIC_EXTRA_PK_LIST.length));
- placeholders.append("?, ?");
- String sqlMetric = new StringBuilder("UPSERT INTO ")
- .append(METRIC_TABLE_NAME).append(" (").append(sqlColumns)
- .append(") VALUES(").append(placeholders).append(")").toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL statement for metric: " + sqlMetric);
- }
- try (PreparedStatement psMetrics = conn.prepareStatement(sqlMetric)) {
- if (metric.getType().equals(TimelineMetric.Type.TIME_SERIES)) {
- LOG.warn("The incoming timeline metric contains time series data, "
- + "which is currently not supported by Phoenix storage. "
- + "Time series will be truncated. ");
- }
- int idx = setStringsForPrimaryKey(psMetrics, context, entity, 1);
- psMetrics.setString(idx++, metric.getId());
- Iterator<Map.Entry<Long, Number>> currNumIter =
- metric.getValues().entrySet().iterator();
- if (currNumIter.hasNext()) {
- // TODO: support time series storage
- Map.Entry<Long, Number> currEntry = currNumIter.next();
- psMetrics.setBytes(idx++,
- GenericObjectMapper.write(currEntry.getValue()));
- psMetrics.setLong(idx++, currEntry.getKey());
- } else {
- psMetrics.setBytes(idx++, GenericObjectMapper.write(null));
- LOG.warn("The incoming metric contains an empty value set. ");
- }
- psMetrics.execute();
- } catch (IOException ie) {
- LOG.error("Exception on converting single data to bytes: "
- + ie.getMessage());
- throw new SQLException(ie);
- }
- }
- }
-
- private static void storeEvents(TimelineEntity entity,
- TimelineCollectorContext context, Connection conn) throws SQLException {
- if (entity.getEvents() == null) {
- return;
- }
- Set<TimelineEvent> events = entity.getEvents();
- for (TimelineEvent event : events) {
- // We need this number to check if the incoming event's info field is empty
- int numPlaceholders = 0;
- StringBuilder sqlColumns = new StringBuilder(
- StringUtils.join(PHOENIX_STORAGE_PK_LIST, ","));
- sqlColumns.append(",")
- .append(StringUtils.join(TIMELINE_EVENT_EXTRA_PK_LIST, ","));
- appendColumnsSQL(sqlColumns, new DynamicColumns<>(
- EVENT_INFO_COLUMN_FAMILY, DynamicColumns.COLUMN_FAMILY_TYPE_BYTES,
- event.getInfo().keySet()));
- numPlaceholders += event.getInfo().keySet().size();
- if (numPlaceholders == 0) {
- continue;
- }
- StringBuilder placeholders = new StringBuilder();
- placeholders.append(
- StringUtils.repeat("?,", PHOENIX_STORAGE_PK_LIST.length))
- .append(StringUtils.repeat("?,", TIMELINE_EVENT_EXTRA_PK_LIST.length));
- // numPlaceholders >= 1 now
- placeholders.append("?")
- .append(StringUtils.repeat(",?", numPlaceholders - 1));
- String sqlEvents = new StringBuilder("UPSERT INTO ")
- .append(EVENT_TABLE_NAME).append(" (").append(sqlColumns)
- .append(") VALUES(").append(placeholders).append(")").toString();
- if (LOG.isDebugEnabled()) {
- LOG.debug("SQL statement for events: " + sqlEvents);
- }
- try (PreparedStatement psEvent = conn.prepareStatement(sqlEvents)) {
- int idx = setStringsForPrimaryKey(psEvent, context, entity, 1);
- psEvent.setLong(idx++, event.getTimestamp());
- psEvent.setString(idx++, event.getId());
- setBytesForColumnFamily(psEvent, event.getInfo(), idx);
- psEvent.execute();
- }
- }
- }
-
- // WARNING: This method will permanently drop a table!
- @Private
- @VisibleForTesting
- void dropTable(String tableName) throws Exception {
- try (Connection conn = getConnection();
- Statement stmt = conn.createStatement()) {
- String sql = "DROP TABLE " + tableName;
- stmt.executeUpdate(sql);
- } catch (SQLException se) {
- LOG.error("Failed in dropping entity table " + se.getLocalizedMessage());
- throw se;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 3a22ed6..5120856 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
@@ -51,6 +53,7 @@ public class TimelineSchemaCreator {
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
+ private static final String PHOENIX_OPTION_SHORT = "p";
public static void main(String[] args) throws Exception {
@@ -83,7 +86,41 @@ public class TimelineSchemaCreator {
hbaseConf.set(ApplicationTable.TABLE_NAME_CONF_NAME,
applicationTableName);
}
- createAllTables(hbaseConf);
+
+ List<Exception> exceptions = new ArrayList<>();
+ try {
+ createAllTables(hbaseConf);
+ LOG.info("Successfully created HBase schema. ");
+ } catch (IOException e) {
+ LOG.error("Error in creating hbase tables: " + e.getMessage());
+ exceptions.add(e);
+ }
+
+ // Create Phoenix data schema if needed
+ if (commandLine.hasOption(PHOENIX_OPTION_SHORT)) {
+ Configuration phoenixConf = new Configuration();
+ try {
+ PhoenixOfflineAggregationWriterImpl phoenixWriter =
+ new PhoenixOfflineAggregationWriterImpl();
+ phoenixWriter.init(phoenixConf);
+ phoenixWriter.start();
+ phoenixWriter.createPhoenixTables();
+ phoenixWriter.stop();
+ LOG.info("Successfully created Phoenix offline aggregation schema. ");
+ } catch (IOException e) {
+ LOG.error("Error in creating phoenix tables: " + e.getMessage());
+ exceptions.add(e);
+ }
+ }
+ if (exceptions.size() > 0) {
+ LOG.warn("Schema creation finished with the following exceptions");
+ for (Exception e : exceptions) {
+ LOG.warn(e.getMessage());
+ }
+ System.exit(-1);
+ } else {
+ LOG.info("Schema creation finished successfully");
+ }
}
/**
@@ -115,6 +152,12 @@ public class TimelineSchemaCreator {
o.setRequired(false);
options.addOption(o);
+ o = new Option(PHOENIX_OPTION_SHORT, "usePhoenix", false,
+ "create Phoenix offline aggregation tables");
+ // No need to set arg name since we do not need an argument here
+ o.setRequired(false);
+ options.addOption(o);
+
CommandLineParser parser = new PosixParser();
CommandLine commandLine = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
new file mode 100644
index 0000000..16c03a3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+
+/**
+ * Class to carry the offline aggregation information for storage level
+ * implementations. There are currently two predefined aggregation info
+ * instances that represent flow and user level offline aggregations. Depend on
+ * its implementation, a storage class may use an OfflineAggregationInfo object
+ * to decide behaviors dynamically.
+ */
+public final class OfflineAggregationInfo {
+ /**
+ * Default flow level aggregation table name
+ */
+ @VisibleForTesting
+ public static final String FLOW_AGGREGATION_TABLE_NAME
+ = "yarn_timeline_flow_aggregation";
+ /**
+ * Default user level aggregation table name
+ */
+ public static final String USER_AGGREGATION_TABLE_NAME
+ = "yarn_timeline_user_aggregation";
+
+ // These lists are not taking effects in table creations.
+ private static final String[] FLOW_AGGREGATION_PK_LIST =
+ { "user", "cluster", "flow_name" };
+ private static final String[] USER_AGGREGATION_PK_LIST = { "user", "cluster"};
+
+ private final String tableName;
+ private final String[] primaryKeyList;
+ private final PrimaryKeyStringSetter primaryKeyStringSetter;
+
+ private OfflineAggregationInfo(String table, String[] pkList,
+ PrimaryKeyStringSetter formatter) {
+ tableName = table;
+ primaryKeyList = pkList;
+ primaryKeyStringSetter = formatter;
+ }
+
+ private interface PrimaryKeyStringSetter {
+ int setValues(PreparedStatement ps, TimelineCollectorContext context,
+ String[] extraInfo, int startPos) throws SQLException;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String[] getPrimaryKeyList() {
+ return primaryKeyList.clone();
+ }
+
+ public int setStringsForPrimaryKey(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo, int startPos)
+ throws SQLException {
+ return primaryKeyStringSetter.setValues(ps, context, extraInfo, startPos);
+ }
+
+ public static final OfflineAggregationInfo FLOW_AGGREGATION =
+ new OfflineAggregationInfo(FLOW_AGGREGATION_TABLE_NAME,
+ FLOW_AGGREGATION_PK_LIST, new PrimaryKeyStringSetter() {
+ @Override
+ public int setValues(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo, int startPos)
+ throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++, context.getClusterId());
+ ps.setString(idx++, context.getFlowName());
+ return idx;
+ }
+ });
+
+ public static final OfflineAggregationInfo USER_AGGREGATION =
+ new OfflineAggregationInfo(USER_AGGREGATION_TABLE_NAME,
+ USER_AGGREGATION_PK_LIST, new PrimaryKeyStringSetter() {
+ @Override
+ public int setValues(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo, int startPos)
+ throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++, context.getClusterId());
+ return idx;
+ }
+ });
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
new file mode 100644
index 0000000..de66a17
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.hbase.IntegrationTestingUtility;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggregationInfo;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ReadOnlyProps;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+public class TestPhoenixOfflineAggregationWriterImpl extends BaseTest {
+ private static PhoenixOfflineAggregationWriterImpl storage;
+ private static final int BATCH_SIZE = 3;
+
+ @BeforeClass
+ public static void setup() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ storage = setupPhoenixClusterAndWriterForTest(conf);
+ }
+
+ @Test(timeout = 90000)
+ public void testFlowLevelAggregationStorage() throws Exception {
+ testAggregator(OfflineAggregationInfo.FLOW_AGGREGATION);
+ }
+
+ @Test(timeout = 90000)
+ public void testUserLevelAggregationStorage() throws Exception {
+ testAggregator(OfflineAggregationInfo.USER_AGGREGATION);
+ }
+
+ @AfterClass
+ public static void cleanup() throws Exception {
+ storage.dropTable(OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME);
+ storage.dropTable(OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME);
+ tearDownMiniCluster();
+ }
+
+ private static PhoenixOfflineAggregationWriterImpl
+ setupPhoenixClusterAndWriterForTest(YarnConfiguration conf)
+ throws Exception{
+ Map<String, String> props = new HashMap<>();
+ // Must update config before starting server
+ props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
+ Boolean.FALSE.toString());
+ props.put("java.security.krb5.realm", "");
+ props.put("java.security.krb5.kdc", "");
+ props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER,
+ Boolean.FALSE.toString());
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
+ props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
+ // Make a small batch size to test multiple calls to reserve sequences
+ props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB,
+ Long.toString(BATCH_SIZE));
+ // Must update config before starting server
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+
+ // Change connection settings for test
+ conf.set(
+ YarnConfiguration.PHOENIX_OFFLINE_STORAGE_CONN_STR,
+ getUrl());
+ PhoenixOfflineAggregationWriterImpl
+ myWriter = new PhoenixOfflineAggregationWriterImpl(TEST_PROPERTIES);
+ myWriter.init(conf);
+ myWriter.start();
+ myWriter.createPhoenixTables();
+ return myWriter;
+ }
+
+ private static TimelineEntity getTestAggregationTimelineEntity() {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "hello1";
+ String type = "testAggregationType";
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(1425016501000L);
+ entity.setModifiedTime(1425016502000L);
+
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("HDFS_BYTES_READ");
+ metric.addValue(1425016501100L, 8000);
+ entity.addMetric(metric);
+
+ return entity;
+ }
+
+ private void testAggregator(OfflineAggregationInfo aggregationInfo)
+ throws Exception {
+ // Set up a list of timeline entities and write them back to Phoenix
+ int numEntity = 1;
+ TimelineEntities te = new TimelineEntities();
+ te.addEntity(getTestAggregationTimelineEntity());
+ TimelineCollectorContext context = new TimelineCollectorContext("cluster_1",
+ "user1", "testFlow", null, 0, null);
+ storage.writeAggregatedEntity(context, te,
+ aggregationInfo);
+
+ // Verify if we're storing all entities
+ String[] primaryKeyList = aggregationInfo.getPrimaryKeyList();
+ String sql = "SELECT COUNT(" + primaryKeyList[primaryKeyList.length - 1]
+ +") FROM " + aggregationInfo.getTableName();
+ verifySQLWithCount(sql, numEntity, "Number of entities should be ");
+ // Check metric
+ sql = "SELECT COUNT(m.HDFS_BYTES_READ) FROM "
+ + aggregationInfo.getTableName() + "(m.HDFS_BYTES_READ VARBINARY) ";
+ verifySQLWithCount(sql, numEntity,
+ "Number of entities with info should be ");
+ }
+
+
+ private void verifySQLWithCount(String sql, int targetCount, String message)
+ throws Exception {
+ try (
+ Statement stmt =
+ storage.getConnection().createStatement();
+ ResultSet rs = stmt.executeQuery(sql)) {
+ assertTrue("Result set empty on statement " + sql, rs.next());
+ assertNotNull("Fail to execute query " + sql, rs);
+ assertEquals(message + " " + targetCount, targetCount, rs.getInt(1));
+ } catch (SQLException se) {
+ fail("SQL exception on query: " + sql
+ + " With exception message: " + se.getLocalizedMessage());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
deleted file mode 100644
index dece83d..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixTimelineWriterImpl.java
+++ /dev/null
@@ -1,152 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
-import org.apache.hadoop.hbase.IntegrationTestingUtility;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
-import org.junit.Test;
-import org.apache.phoenix.hbase.index.write.IndexWriterUtils;
-import org.apache.phoenix.query.BaseTest;
-import org.apache.phoenix.query.QueryServices;
-import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.ReadOnlyProps;
-
-import java.sql.ResultSet;
-import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.HashMap;
-import java.util.Map;
-
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-
-public class TestPhoenixTimelineWriterImpl extends BaseTest {
- private static PhoenixTimelineWriterImpl writer;
- private static final int BATCH_SIZE = 3;
-
- @BeforeClass
- public static void setup() throws Exception {
- YarnConfiguration conf = new YarnConfiguration();
- writer = setupPhoenixClusterAndWriterForTest(conf);
- }
-
- @Test(timeout = 90000)
- public void testPhoenixWriterBasic() throws Exception {
- // Set up a list of timeline entities and write them back to Phoenix
- int numEntity = 12;
- TimelineEntities te =
- TestTimelineWriterImpl.getStandardTestTimelineEntities(numEntity);
- writer.write("cluster_1", "user1", "testFlow", "version1", 1l, "app_test_1", te);
- // Verify if we're storing all entities
- String sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME;
- verifySQLWithCount(sql, numEntity, "Number of entities should be ");
- // Check config (half of all entities)
- sql = "SELECT COUNT(c.config) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(c.config VARCHAR) ";
- verifySQLWithCount(sql, (numEntity / 2),
- "Number of entities with config should be ");
- // Check info (half of all entities)
- sql = "SELECT COUNT(i.info1) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME + "(i.info1 VARBINARY) ";
- verifySQLWithCount(sql, (numEntity / 2),
- "Number of entities with info should be ");
- // Check config and info (a quarter of all entities)
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
- + "(c.config VARCHAR, i.info1 VARBINARY) "
- + "WHERE c.config IS NOT NULL AND i.info1 IS NOT NULL";
- verifySQLWithCount(sql, (numEntity / 4),
- "Number of entities with both config and info should be ");
- // Check relatesToEntities and isRelatedToEntities
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME
- + "(rt.testType VARCHAR, ir.testType VARCHAR) "
- + "WHERE rt.testType IS NOT NULL AND ir.testType IS NOT NULL";
- verifySQLWithCount(sql, numEntity - 2,
- "Number of entities with both relatesTo and isRelatedTo should be ");
- // Check event
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.EVENT_TABLE_NAME;
- verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
- // Check metrics
- sql = "SELECT COUNT(entity_id) FROM "
- + PhoenixTimelineWriterImpl.METRIC_TABLE_NAME;
- verifySQLWithCount(sql, (numEntity / 4), "Number of events should be ");
- }
-
- @AfterClass
- public static void cleanup() throws Exception {
- writer.dropTable(PhoenixTimelineWriterImpl.ENTITY_TABLE_NAME);
- writer.dropTable(PhoenixTimelineWriterImpl.EVENT_TABLE_NAME);
- writer.dropTable(PhoenixTimelineWriterImpl.METRIC_TABLE_NAME);
- writer.serviceStop();
- tearDownMiniCluster();
- }
-
- private static PhoenixTimelineWriterImpl setupPhoenixClusterAndWriterForTest(
- YarnConfiguration conf) throws Exception{
- Map<String, String> props = new HashMap<>();
- // Must update config before starting server
- props.put(QueryServices.STATS_USE_CURRENT_TIME_ATTRIB,
- Boolean.FALSE.toString());
- props.put("java.security.krb5.realm", "");
- props.put("java.security.krb5.kdc", "");
- props.put(IntegrationTestingUtility.IS_DISTRIBUTED_CLUSTER,
- Boolean.FALSE.toString());
- props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
- props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
- // Make a small batch size to test multiple calls to reserve sequences
- props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB,
- Long.toString(BATCH_SIZE));
- // Must update config before starting server
- setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
-
- PhoenixTimelineWriterImpl myWriter = new PhoenixTimelineWriterImpl();
- // Change connection settings for test
- conf.set(
- PhoenixTimelineWriterImpl.TIMELINE_SERVICE_PHOENIX_STORAGE_CONN_STR,
- getUrl());
- myWriter.connProperties = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- myWriter.serviceInit(conf);
- return myWriter;
- }
-
- private void verifySQLWithCount(String sql, int targetCount, String message)
- throws Exception {
- try (
- Statement stmt =
- writer.getConnection().createStatement();
- ResultSet rs = stmt.executeQuery(sql)) {
- assertTrue("Result set empty on statement " + sql, rs.next());
- assertNotNull("Fail to execute query " + sql, rs);
- assertEquals(message + " " + targetCount, targetCount, rs.getInt(1));
- } catch (SQLException se) {
- fail("SQL exception on query: " + sql
- + " With exception message: " + se.getLocalizedMessage());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97f211b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
deleted file mode 100644
index 7a7afc0..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestTimelineWriterImpl.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-
-public class TestTimelineWriterImpl {
- static TimelineEntities getStandardTestTimelineEntities(int listSize) {
- TimelineEntities te = new TimelineEntities();
- for (int i = 0; i < listSize; i++) {
- TimelineEntity entity = new TimelineEntity();
- String id = "hello" + i;
- String type = "testType";
- entity.setId(id);
- entity.setType(type);
- entity.setCreatedTime(1425016501000L + i);
- entity.setModifiedTime(1425016502000L + i);
- if (i > 0) {
- entity.addRelatesToEntity(type, "hello" + i);
- entity.addRelatesToEntity(type, "hello" + (i - 1));
- }
- if (i < listSize - 1) {
- entity.addIsRelatedToEntity(type, "hello" + i);
- entity.addIsRelatedToEntity(type, "hello" + (i + 1));
- }
- int category = i % 4;
- switch (category) {
- case 0:
- entity.addConfig("config", "config" + i);
- // Fall through deliberately
- case 1:
- entity.addInfo("info1", new Integer(i));
- entity.addInfo("info2", "helloworld");
- // Fall through deliberately
- case 2:
- break;
- case 3:
- entity.addConfig("config", "config" + i);
- TimelineEvent event = new TimelineEvent();
- event.setId("test event");
- event.setTimestamp(1425016501100L + i);
- event.addInfo("test_info", "content for " + entity.getId());
- event.addInfo("test_info1", new Integer(i));
- entity.addEvent(event);
- TimelineMetric metric = new TimelineMetric();
- metric.setId("HDFS_BYTES_READ");
- metric.addValue(1425016501100L + i, 8000 + i);
- entity.addMetric(metric);
- break;
- }
- te.addEntity(entity);
- }
- return te;
- }
-}
[33/50] [abbrv] hadoop git commit: YARN-3836. add equals and hashCode
to TimelineEntity and other classes in the data model (Li Lu via sjlee)
Posted by sj...@apache.org.
YARN-3836. add equals and hashCode to TimelineEntity and other classes in the data model (Li Lu via sjlee)
(cherry picked from commit 2d4a8f4563c06339717ca9410b2794754603fba3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cced5944
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cced5944
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cced5944
Branch: refs/heads/YARN-2928
Commit: cced59444b880ed60bf5f288ef692ca6fca198ad
Parents: e281987
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Jul 9 20:50:48 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Tue Aug 25 10:47:14 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/timelineservice/TimelineEntity.java | 89 +++++++++++++++++++-
.../records/timelineservice/TimelineEvent.java | 41 ++++++++-
.../records/timelineservice/TimelineMetric.java | 30 +++++++
.../TestTimelineServiceRecords.java | 36 +++++++-
5 files changed, 195 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cced5944/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d77ad59..6e10926 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -87,6 +87,9 @@ Branch YARN-2928: Timeline Server Next Generation: Phase 1
YARN-3706. Generalize native HBase writer for additional tables (Joep
Rottinghuis via sjlee)
+ YARN-3836. add equals and hashCode to TimelineEntity and other classes in
+ the data model (Li Lu via sjlee)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cced5944/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 60fba85..9ef2d90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -31,11 +31,25 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+/**
+ * The basic timeline entity data structure for timeline service v2. Timeline
+ * entity objects are not thread safe and should not be accessed concurrently.
+ * All collection members will be initialized into empty collections. Two
+ * timeline entities are equal iff. their type and id are identical.
+ *
+ * All non-primitive type, non-collection members will be initialized into null.
+ * User should set the type and id of a timeline entity to make it valid (can be
+ * checked by using the {@link #isValid()} method). Callers to the getters
+ * should perform null checks for non-primitive type, non-collection members.
+ *
+ * Callers are recommended not to alter the returned collection objects from the
+ * getters.
+ */
@XmlRootElement(name = "entity")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public class TimelineEntity {
+public class TimelineEntity implements Comparable<TimelineEntity> {
protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_";
@XmlRootElement(name = "identifier")
@@ -77,6 +91,41 @@ public class TimelineEntity {
"type='" + type + '\'' +
", id='" + id + '\'' + "]";
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((id == null) ? 0 : id.hashCode());
+ result =
+ prime * result + ((type == null) ? 0 : type.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (!(obj instanceof Identifier)) {
+ return false;
+ }
+ Identifier other = (Identifier) obj;
+ if (id == null) {
+ if (other.getId() != null) {
+ return false;
+ }
+ } else if (!id.equals(other.getId())) {
+ return false;
+ }
+ if (type == null) {
+ if (other.getType() != null) {
+ return false;
+ }
+ } else if (!type.equals(other.getType())) {
+ return false;
+ }
+ return true;
+ }
}
private TimelineEntity real;
@@ -471,6 +520,44 @@ public class TimelineEntity {
}
}
+ public boolean isValid() {
+ return (getId() != null && getType() != null);
+ }
+
+ // When get hashCode for a timeline entity, or check if two timeline entities
+ // are equal, we only compare their identifiers (id and type)
+ @Override
+ public int hashCode() {
+ return getIdentifier().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (!(obj instanceof TimelineEntity))
+ return false;
+ TimelineEntity other = (TimelineEntity) obj;
+ return getIdentifier().equals(other.getIdentifier());
+ }
+
+ @Override
+ public int compareTo(TimelineEntity other) {
+ int comparison = getType().compareTo(other.getType());
+ if (comparison == 0) {
+ if (getCreatedTime() > other.getCreatedTime()) {
+ // Order by created time desc
+ return -1;
+ } else if (getCreatedTime() < other.getCreatedTime()) {
+ return 1;
+ } else {
+ return getId().compareTo(other.getId());
+ }
+ } else {
+ return comparison;
+ }
+ }
+
protected TimelineEntity getReal() {
return real == null ? this : real;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cced5944/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index b6b82a7..1dbf7e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -32,7 +32,7 @@ import java.util.Map;
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public class TimelineEvent {
+public class TimelineEvent implements Comparable<TimelineEvent> {
private String id;
private HashMap<String, Object> info = new HashMap<>();
private long timestamp;
@@ -81,4 +81,43 @@ public class TimelineEvent {
public void setTimestamp(long timestamp) {
this.timestamp = timestamp;
}
+
+ public boolean isValid() {
+ return (id != null && timestamp != 0L);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = (int) (timestamp ^ (timestamp >>> 32));
+ result = 31 * result + id.hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof TimelineEvent))
+ return false;
+
+ TimelineEvent event = (TimelineEvent) o;
+
+ if (timestamp != event.timestamp)
+ return false;
+ if (!id.equals(event.id)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public int compareTo(TimelineEvent other) {
+ if (timestamp > other.timestamp) {
+ return -1;
+ } else if (timestamp < other.timestamp) {
+ return 1;
+ } else {
+ return id.compareTo(other.id);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cced5944/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 172b56c..f44c1a2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -124,4 +124,34 @@ public class TimelineMetric {
this.values.clear();
this.values.putAll(values);
}
+
+ public boolean isValid() {
+ return (id != null);
+ }
+
+ @Override
+ public int hashCode() {
+ int result = id.hashCode();
+ result = 31 * result + type.hashCode();
+ return result;
+ }
+
+ // Only check if type and id are equal
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (!(o instanceof TimelineMetric))
+ return false;
+
+ TimelineMetric m = (TimelineMetric) o;
+
+ if (!id.equals(m.id)) {
+ return false;
+ }
+ if (type != m.type) {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cced5944/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 95b922e..78943e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -100,6 +100,13 @@ public class TestTimelineServiceRecords {
}
entity.addMetric(metric2);
+ TimelineMetric metric3 = new TimelineMetric(TimelineMetric.Type.SINGLE_VALUE);
+ metric3.setId("test metric id 1");
+ metric3.addValue(4L, (short) 4);
+ Assert.assertEquals("metric3 should equal to metric2! ", metric3, metric2);
+ Assert.assertNotEquals("metric1 should not equal to metric2! ",
+ metric1, metric2);
+
TimelineEvent event1 = new TimelineEvent();
event1.setId("test event id 1");
event1.addInfo("test info key 1", "test info value 1");
@@ -108,7 +115,7 @@ public class TestTimelineServiceRecords {
event1.addInfo("test info key 3", true);
Assert.assertTrue(
event1.getInfo().get("test info key 3") instanceof Boolean);
- event1.setTimestamp(0L);
+ event1.setTimestamp(1L);
entity.addEvent(event1);
TimelineEvent event2 = new TimelineEvent();
@@ -119,9 +126,18 @@ public class TestTimelineServiceRecords {
event2.addInfo("test info key 3", true);
Assert.assertTrue(
event2.getInfo().get("test info key 3") instanceof Boolean);
- event2.setTimestamp(1L);
+ event2.setTimestamp(2L);
entity.addEvent(event2);
+ Assert.assertFalse("event1 should not equal to event2! ",
+ event1.equals(event2));
+ TimelineEvent event3 = new TimelineEvent();
+ event3.setId("test event id 1");
+ event3.setTimestamp(1L);
+ Assert.assertEquals("event1 should equal to event3! ", event3, event1);
+ Assert.assertNotEquals("event1 should not equal to event2! ",
+ event1, event2);
+
entity.setCreatedTime(0L);
entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
@@ -136,6 +152,22 @@ public class TestTimelineServiceRecords {
TimelineEntity entity2 = new TimelineEntity();
entities.addEntity(entity2);
LOG.info(TimelineUtils.dumpTimelineRecordtoJSON(entities, true));
+
+ Assert.assertFalse("entity 1 should not be valid without type and id",
+ entity1.isValid());
+ entity1.setId("test id 2");
+ entity1.setType("test type 2");
+ entity2.setId("test id 1");
+ entity2.setType("test type 1");
+
+ Assert.assertEquals("Timeline entity should equal to entity2! ",
+ entity, entity2);
+ Assert.assertNotEquals("entity1 should not equal to entity! ",
+ entity1, entity);
+ Assert.assertEquals("entity should be less than entity1! ",
+ entity1.compareTo(entity), 1);
+ Assert.assertEquals("entity's hash code should be -28727840 but not "
+ + entity.hashCode(), entity.hashCode(), -28727840);
}
@Test
[31/50] [abbrv] hadoop git commit: YARN-3706. Generalize native HBase
writer for additional tables (Joep Rottinghuis via sjlee)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
new file mode 100644
index 0000000..ee57890
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -0,0 +1,303 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * Used to separate row qualifiers, column qualifiers and compount fields.
+ */
+public enum Separator {
+
+ /**
+ * separator in key or column qualifier fields
+ */
+ QUALIFIERS("!", "%0$"),
+
+ /**
+ * separator in values, and/or compound key/column qualifier fields.
+ */
+ VALUES("?", "%1$"),
+
+ /**
+ * separator in values, often used to avoid having these in qualifiers and
+ * names. Note that if we use HTML form encoding through URLEncoder, we end up
+ * getting a + for a space, which may already occur in strings, so we don't
+ * want that.
+ */
+ SPACE(" ", "%2$");
+
+ /**
+ * The string value of this separator.
+ */
+ private final String value;
+
+ /**
+ * The URLEncoded version of this separator
+ */
+ private final String encodedValue;
+
+ /**
+ * The bye representation of value.
+ */
+ private final byte[] bytes;
+
+ /**
+ * The value quoted so that it can be used as a safe regex
+ */
+ private final String quotedValue;
+
+ private static final byte[] EMPTY_BYTES = new byte[0];
+
+ /**
+ * @param value of the separator to use. Cannot be null or empty string.
+ * @param encodedValue choose something that isn't likely to occur in the data
+ * itself. Cannot be null or empty string.
+ */
+ private Separator(String value, String encodedValue) {
+ this.value = value;
+ this.encodedValue = encodedValue;
+
+ // validation
+ if (value == null || value.length() == 0 || encodedValue == null
+ || encodedValue.length() == 0) {
+ throw new IllegalArgumentException(
+ "Cannot create separator from null or empty string.");
+ }
+
+ this.bytes = Bytes.toBytes(value);
+ this.quotedValue = Pattern.quote(value);
+ }
+
+ /**
+ * Used to make token safe to be used with this separator without collisions.
+ *
+ * @param token
+ * @return the token with any occurrences of this separator URLEncoded.
+ */
+ public String encode(String token) {
+ if (token == null || token.length() == 0) {
+ // Nothing to replace
+ return token;
+ }
+ return token.replace(value, encodedValue);
+ }
+
+ /**
+ * @param token
+ * @return the token with any occurrences of the encoded separator replaced by
+ * the separator itself.
+ */
+ public String decode(String token) {
+ if (token == null || token.length() == 0) {
+ // Nothing to replace
+ return token;
+ }
+ return token.replace(encodedValue, value);
+ }
+
+ /**
+ * Encode the given separators in the token with their encoding equivalent.
+ * This means that when encoding is already present in the token itself, this
+ * is not a reversible process. See also {@link #decode(String, Separator...)}
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return non-null byte representation of the token with occurrences of the
+ * separators encoded.
+ */
+ public static byte[] encode(String token, Separator... separators) {
+ if (token == null) {
+ return EMPTY_BYTES;
+ }
+ String result = token;
+ for (Separator separator : separators) {
+ if (separator != null) {
+ result = separator.encode(result);
+ }
+ }
+ return Bytes.toBytes(result);
+ }
+
+ /**
+ * Decode the given separators in the token with their decoding equivalent.
+ * This means that when encoding is already present in the token itself, this
+ * is not a reversible process.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return String representation of the token with occurrences of the URL
+ * encoded separators decoded.
+ */
+ public static String decode(byte[] token, Separator... separators) {
+ if (token == null) {
+ return null;
+ }
+ return decode(Bytes.toString(token), separators);
+ }
+
+ /**
+ * Decode the given separators in the token with their decoding equivalent.
+ * This means that when encoding is already present in the token itself, this
+ * is not a reversible process.
+ *
+ * @param token containing possible separators that need to be encoded.
+ * @param separators to be encoded in the token with their URLEncoding
+ * equivalent.
+ * @return String representation of the token with occurrences of the URL
+ * encoded separators decoded.
+ */
+ public static String decode(String token, Separator... separators) {
+ if (token == null) {
+ return null;
+ }
+ String result = token;
+ for (Separator separator : separators) {
+ if (separator != null) {
+ result = separator.decode(result);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Returns a single byte array containing all of the individual arrays
+ * components separated by this separator.
+ *
+ * @param components
+ * @return byte array after joining the components
+ */
+ public byte[] join(byte[]... components) {
+ if (components == null || components.length == 0) {
+ return EMPTY_BYTES;
+ }
+
+ int finalSize = 0;
+ finalSize = this.value.length() * (components.length - 1);
+ for (byte[] comp : components) {
+ if (comp != null) {
+ finalSize += comp.length;
+ }
+ }
+
+ byte[] buf = new byte[finalSize];
+ int offset = 0;
+ for (int i = 0; i < components.length; i++) {
+ if (components[i] != null) {
+ System.arraycopy(components[i], 0, buf, offset, components[i].length);
+ offset += components[i].length;
+ }
+ if (i < (components.length - 1)) {
+ System.arraycopy(this.bytes, 0, buf, offset, this.value.length());
+ offset += this.value.length();
+ }
+
+ }
+ return buf;
+ }
+
+ /**
+ * Concatenates items (as String), using this separator.
+ *
+ * @param items Items join, {@code toString()} will be called in each item.
+ * Any occurrence of the separator in the individual strings will be
+ * first encoded. Cannot be null.
+ * @return non-null joined result. Note that when separator is {@literal null}
+ * the result is simply all items concatenated and the process is not
+ * reversible through {@link #splitEncoded(String)}
+ */
+ public String joinEncoded(String... items) {
+ if (items == null || items.length == 0) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder(encode(items[0].toString()));
+ // Start at 1, we've already grabbed the first value at index 0
+ for (int i = 1; i < items.length; i++) {
+ sb.append(this.value);
+ sb.append(encode(items[i].toString()));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Concatenates items (as String), using this separator.
+ *
+ * @param items Items join, {@code toString()} will be called in each item.
+ * Any occurrence of the separator in the individual strings will be
+ * first encoded. Cannot be null.
+ * @return non-null joined result. Note that when separator is {@literal null}
+ * the result is simply all items concatenated and the process is not
+ * reversible through {@link #splitEncoded(String)}
+ */
+ public String joinEncoded(Iterable<?> items) {
+ if (items == null) {
+ return "";
+ }
+ Iterator<?> i = items.iterator();
+ if (!i.hasNext()) {
+ return "";
+ }
+
+ StringBuilder sb = new StringBuilder(encode(i.next().toString()));
+ while (i.hasNext()) {
+ sb.append(this.value);
+ sb.append(encode(i.next().toString()));
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * @param compoundValue containing individual values separated by this
+ * separator, which have that separator encoded.
+ * @return non-null set of values from the compoundValue with the separator
+ * decoded.
+ */
+ public Collection<String> splitEncoded(String compoundValue) {
+ List<String> result = new ArrayList<String>();
+ if (compoundValue != null) {
+ for (String value : compoundValue.split(quotedValue)) {
+ result.add(decode(value));
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Splits the source array into multiple array segments using this separator,
+ * up to a maximum of count items. This will naturally produce copied byte
+ * arrays for each of the split segments.
+ * @param source to be split
+ * @param limit on how many segments are supposed to be returned. Negative
+ * value indicates no limit on number of segments.
+ * @return source split by this separator.
+ */
+ public byte[][] split(byte[] source, int limit) {
+ return TimelineWriterUtils.split(source, this.bytes, limit);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
new file mode 100644
index 0000000..5518a27
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntitySchemaConstants.java
@@ -0,0 +1,68 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * contains the constants used in the context of schema accesses for
+ * {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
+ * information
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class TimelineEntitySchemaConstants {
+
+ /**
+ * Used to create a pre-split for tables starting with a username in the
+ * prefix. TODO: this may have to become a config variable (string with
+ * separators) so that different installations can presplit based on their own
+ * commonly occurring names.
+ */
+ private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
+ Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
+ Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
+ Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
+ Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
+ Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
+ Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
+ Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
+ Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
+ Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
+ Bytes.toBytes("z") };
+
+ /**
+ * The length at which keys auto-split
+ */
+ public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
+
+ /**
+ * @return splits for splits where a user is a prefix.
+ */
+ public final static byte[][] getUsernameSplits() {
+ byte[][] kloon = USERNAME_SPLITS.clone();
+ // Deep copy.
+ for (int row = 0; row < USERNAME_SPLITS.length; row++) {
+ kloon[row] = Bytes.copy(USERNAME_SPLITS[row]);
+ }
+ return kloon;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
new file mode 100644
index 0000000..28a0b6a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineWriterUtils.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * bunch of utility functions used across TimelineWriter classes
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class TimelineWriterUtils {
+
+ /** empty bytes */
+ public static final byte[] EMPTY_BYTES = new byte[0];
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @return byte[] array after splitting the source
+ */
+ public static byte[][] split(byte[] source, byte[] separator) {
+ return split(source, separator, -1);
+ }
+
+ /**
+ * Splits the source array into multiple array segments using the given
+ * separator, up to a maximum of count items. This will naturally produce
+ * copied byte arrays for each of the split segments. To identify the split
+ * ranges without the array copies, see
+ * {@link TimelineWriterUtils#splitRanges(byte[], byte[])}.
+ *
+ * @param source
+ * @param separator
+ * @param limit a negative value indicates no limit on number of segments.
+ * @return byte[][] after splitting the input source
+ */
+ public static byte[][] split(byte[] source, byte[] separator, int limit) {
+ List<Range> segments = splitRanges(source, separator, limit);
+
+ byte[][] splits = new byte[segments.size()][];
+ for (int i = 0; i < segments.size(); i++) {
+ Range r = segments.get(i);
+ byte[] tmp = new byte[r.length()];
+ if (tmp.length > 0) {
+ System.arraycopy(source, r.start(), tmp, 0, r.length());
+ }
+ splits[i] = tmp;
+ }
+ return splits;
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator) {
+ return splitRanges(source, separator, -1);
+ }
+
+ /**
+ * Returns a list of ranges identifying [start, end) -- closed, open --
+ * positions within the source byte array that would be split using the
+ * separator byte array.
+ *
+ * @param source the source data
+ * @param separator the separator pattern to look for
+ * @param limit the maximum number of splits to identify in the source
+ */
+ public static List<Range> splitRanges(byte[] source, byte[] separator,
+ int limit) {
+ List<Range> segments = new ArrayList<Range>();
+ if ((source == null) || (separator == null)) {
+ return segments;
+ }
+ int start = 0;
+ itersource: for (int i = 0; i < source.length; i++) {
+ for (int j = 0; j < separator.length; j++) {
+ if (source[i + j] != separator[j]) {
+ continue itersource;
+ }
+ }
+ // all separator elements matched
+ if (limit > 0 && segments.size() >= (limit - 1)) {
+ // everything else goes in one final segment
+ break;
+ }
+
+ segments.add(new Range(start, i));
+ start = i + separator.length;
+ // i will be incremented again in outer for loop
+ i += separator.length - 1;
+ }
+ // add in remaining to a final range
+ if (start <= source.length) {
+ segments.add(new Range(start, source.length));
+ }
+ return segments;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
new file mode 100644
index 0000000..64a11f8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TypedBufferedMutator.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.hbase.client.BufferedMutator;
+
+/**
+ * Just a typed wrapper around {@link BufferedMutator} used to ensure that
+ * columns can write only to the table mutator for the right table.
+ */
+public interface TypedBufferedMutator<T> extends BufferedMutator {
+ // This class is intentionally left (almost) blank
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
new file mode 100644
index 0000000..32577fb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
new file mode 100644
index 0000000..90da966
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -0,0 +1,141 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies fully qualified columns for the {@link EntityTable}.
+ */
+public enum EntityColumn implements Column<EntityTable> {
+
+ /**
+ * Identifier for the entity.
+ */
+ ID(EntityColumnFamily.INFO, "id"),
+
+ /**
+ * The type of entity
+ */
+ TYPE(EntityColumnFamily.INFO, "type"),
+
+ /**
+ * When the entity was created.
+ */
+ CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
+
+ /**
+ * When it was modified.
+ */
+ MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
+
+ /**
+ * The version of the flow that this entity belongs to.
+ */
+ FLOW_VERSION(EntityColumnFamily.INFO, "flow_version");
+
+ private final ColumnHelper<EntityTable> column;
+ private final ColumnFamily<EntityTable> columnFamily;
+ private final String columnQualifier;
+ private final byte[] columnQualifierBytes;
+
+ private EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ String columnQualifier) {
+ this.columnFamily = columnFamily;
+ this.columnQualifier = columnQualifier;
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnQualifierBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
+ this.column = new ColumnHelper<EntityTable>(columnFamily);
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnQualifier() {
+ return columnQualifier;
+ }
+
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<EntityTable> tableMutator, Long timestamp,
+ Object inputValue) throws IOException {
+ column.store(rowKey, tableMutator, columnQualifierBytes, timestamp,
+ inputValue);
+ }
+
+ public Object readResult(Result result) throws IOException {
+ return column.readResult(result, columnQualifierBytes);
+ }
+
+ /**
+ * Retrieve an {@link EntityColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnQualifier Name of the column to retrieve
+ * @return the corresponding {@link EntityColumn} or null
+ */
+ public static final EntityColumn columnFor(String columnQualifier) {
+
+ // Match column based on value, assume column family matches.
+ for (EntityColumn ec : EntityColumn.values()) {
+ // Find a match based only on name.
+ if (ec.getColumnQualifier().equals(columnQualifier)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link EntityColumn} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code a.equals(b) & x.equals(y)} or
+ * {@code (x == y == null)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param name Name of the column to retrieve
+ * @return the corresponding {@link EntityColumn} or null if both arguments
+ * don't match.
+ */
+ public static final EntityColumn columnFor(EntityColumnFamily columnFamily,
+ String name) {
+
+ for (EntityColumn ec : EntityColumn.values()) {
+ // Find a match based column family and on name.
+ if (ec.columnFamily.equals(columnFamily)
+ && ec.getColumnQualifier().equals(name)) {
+ return ec;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
new file mode 100644
index 0000000..8a95d12
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnFamily.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents the entity table column families.
+ */
+public enum EntityColumnFamily implements ColumnFamily<EntityTable> {
+
+ /**
+ * Info column family houses known columns, specifically ones included in
+ * columnfamily filters.
+ */
+ INFO("i"),
+
+ /**
+ * Configurations are in a separate column family for two reasons: a) the size
+ * of the config values can be very large and b) we expect that config values
+ * are often separately accessed from other metrics and info columns.
+ */
+ CONFIGS("c"),
+
+ /**
+ * Metrics have a separate column family, because they have a separate TTL.
+ */
+ METRICS("m");
+
+ /**
+ * Byte representation of this column family.
+ */
+ private final byte[] bytes;
+
+ /**
+ * @param value create a column family with this name. Must be lower case and
+ * without spaces.
+ */
+ private EntityColumnFamily(String value) {
+ // column families should be lower case and not contain any spaces.
+ this.bytes = Bytes.toBytes(Separator.SPACE.encode(value));
+ }
+
+ public byte[] getBytes() {
+ return Bytes.copy(bytes);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
new file mode 100644
index 0000000..4459868
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.NavigableMap;
+
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+
+/**
+ * Identifies partially qualified columns for the entity table.
+ */
+public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
+
+ /**
+ * To store TimelineEntity getIsRelatedToEntities values.
+ */
+ IS_RELATED_TO(EntityColumnFamily.INFO, "s"),
+
+ /**
+ * To store TimelineEntity getRelatesToEntities values.
+ */
+ RELATES_TO(EntityColumnFamily.INFO, "r"),
+
+ /**
+ * Lifecycle events for an entity
+ */
+ EVENT(EntityColumnFamily.INFO, "e"),
+
+ /**
+ * Config column stores configuration with config key as the column name.
+ */
+ CONFIG(EntityColumnFamily.CONFIGS, null),
+
+ /**
+ * Metrics are stored with the metric name as the column name.
+ */
+ METRIC(EntityColumnFamily.METRICS, null);
+
+ private final ColumnHelper<EntityTable> column;
+ private final ColumnFamily<EntityTable> columnFamily;
+
+ /**
+ * Can be null for those cases where the provided column qualifier is the
+ * entire column name.
+ */
+ private final String columnPrefix;
+ private final byte[] columnPrefixBytes;
+
+ /**
+ * Private constructor, meant to be used by the enum definition.
+ *
+ * @param columnFamily that this column is stored in.
+ * @param columnPrefix for this column.
+ */
+ private EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix) {
+ column = new ColumnHelper<EntityTable>(columnFamily);
+ this.columnFamily = columnFamily;
+ this.columnPrefix = columnPrefix;
+ if (columnPrefix == null) {
+ this.columnPrefixBytes = null;
+ } else {
+ // Future-proof by ensuring the right column prefix hygiene.
+ this.columnPrefixBytes =
+ Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
+ }
+ }
+
+ /**
+ * @return the column name value
+ */
+ private String getColumnPrefix() {
+ return columnPrefix;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #store(byte[],
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.
+ * TypedBufferedMutator, java.lang.String, java.lang.Long, java.lang.Object)
+ */
+ public void store(byte[] rowKey,
+ TypedBufferedMutator<EntityTable> tableMutator, String qualifier,
+ Long timestamp, Object inputValue) throws IOException {
+
+ // Null check
+ if (qualifier == null) {
+ throw new IOException("Cannot store column with null qualifier in "
+ + tableMutator.getName().getNameAsString());
+ }
+
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+
+ column.store(rowKey, tableMutator, columnQualifier, timestamp, inputValue);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResult(org.apache.hadoop.hbase.client.Result, java.lang.String)
+ */
+ public Object readResult(Result result, String qualifier) throws IOException {
+ byte[] columnQualifier =
+ ColumnHelper.getColumnQualifier(this.columnPrefixBytes, qualifier);
+ return column.readResult(result, columnQualifier);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public Map<String, Object> readResults(Result result) throws IOException {
+ return column.readResults(result, columnPrefixBytes);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
+ * #readTimeseriesResults(org.apache.hadoop.hbase.client.Result)
+ */
+ public NavigableMap<String, NavigableMap<Long, Number>> readTimeseriesResults(
+ Result result) throws IOException {
+ return column.readTimeseriesResults(result, columnPrefixBytes);
+ }
+
+ /**
+ * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(x) == columnFor(y)} if
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ *
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link EntityColumnPrefix} or null
+ */
+ public static final EntityColumnPrefix columnFor(String columnPrefix) {
+
+ // Match column based on value, assume column family matches.
+ for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) {
+ // Find a match based only on name.
+ if (ecp.getColumnPrefix().equals(columnPrefix)) {
+ return ecp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+ /**
+ * Retrieve an {@link EntityColumnPrefix} given a name, or null if there is no
+ * match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
+ * if and only if {@code (x == y == null)} or
+ * {@code a.equals(b) & x.equals(y)}
+ *
+ * @param columnFamily The columnFamily for which to retrieve the column.
+ * @param columnPrefix Name of the column to retrieve
+ * @return the corresponding {@link EntityColumnPrefix} or null if both
+ * arguments don't match.
+ */
+ public static final EntityColumnPrefix columnFor(
+ EntityColumnFamily columnFamily, String columnPrefix) {
+
+ // TODO: needs unit test to confirm and need to update javadoc to explain
+ // null prefix case.
+
+ for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) {
+ // Find a match based column family and on name.
+ if (ecp.columnFamily.equals(columnFamily)
+ && (((columnPrefix == null) && (ecp.getColumnPrefix() == null)) || (ecp
+ .getColumnPrefix().equals(columnPrefix)))) {
+ return ecp;
+ }
+ }
+
+ // Default to null
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
new file mode 100644
index 0000000..61958c2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+
+/**
+ * Represents a rowkey for the entity table.
+ */
+public class EntityRowKey {
+ // TODO: more methods are needed for this class.
+
+ // TODO: API needs to be cleaned up.
+
+ /**
+ * Constructs a row key prefix for the entity table as follows:
+ * {@code userName!clusterId!flowId!flowRunId!AppId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKeyPrefix(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] third = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId));
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Constructs a row key prefix for the entity table as follows:
+ * {@code userName!clusterId!flowId!flowRunId!AppId}
+ *
+ * @param clusterId
+ * @param userId
+ * @param flowId
+ * @param flowRunId
+ * @param appId
+ * @return byte array with the row key prefix
+ */
+ public static byte[] getRowKey(String clusterId, String userId,
+ String flowId, Long flowRunId, String appId, TimelineEntity te) {
+ byte[] first =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
+ flowId));
+ // Note that flowRunId is a long, so we can't encode them all at the same
+ // time.
+ byte[] second = Bytes.toBytes(EntityRowKey.invert(flowRunId));
+ byte[] third =
+ Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(appId, te.getType(),
+ te.getId()));
+ return Separator.QUALIFIERS.join(first, second, third);
+ }
+
+ /**
+ * Converts a timestamp into it's inverse timestamp to be used in (row) keys
+ * where we want to have the most recent timestamp in the top of the table
+ * (scans start at the most recent timestamp first).
+ *
+ * @param key value to be inverted so that the latest version will be first in
+ * a scan.
+ * @return inverted long
+ */
+ public static long invert(Long key) {
+ return Long.MAX_VALUE - key;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
new file mode 100644
index 0000000..61f7c4c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import java.io.IOException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.regionserver.BloomType;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineEntitySchemaConstants;
+
+/**
+ * The entity table as column families info, config and metrics. Info stores
+ * information about a timeline entity object config stores configuration data
+ * of a timeline entity object metrics stores the metrics of a timeline entity
+ * object
+ *
+ * Example entity table record:
+ *
+ * <pre>
+ * |--------------------------------------------------------------------|
+ * | Row | Column Family | Column Family| Column Family|
+ * | key | info | metrics | config |
+ * |--------------------------------------------------------------------|
+ * | userName! | id:entityId | metricId1: | configKey1: |
+ * | clusterId! | | metricValue1 | configValue1 |
+ * | flowId! | type:entityType | @timestamp1 | |
+ * | flowRunId! | | | configKey2: |
+ * | AppId! | created_time: | metriciD1: | configValue2 |
+ * | entityType!| 1392993084018 | metricValue2 | |
+ * | entityId | | @timestamp2 | |
+ * | | modified_time: | | |
+ * | | 1392995081012 | metricId2: | |
+ * | | | metricValue1 | |
+ * | | r!relatesToKey: | @timestamp2 | |
+ * | | id3?id4?id5 | | |
+ * | | | | |
+ * | | s!isRelatedToKey | | |
+ * | | id7?id9?id6 | | |
+ * | | | | |
+ * | | e!eventId?eventInfoKey: | | |
+ * | | eventInfoValue | | |
+ * | | | | |
+ * | | flowVersion: | | |
+ * | | versionValue | | |
+ * |--------------------------------------------------------------------|
+ * </pre>
+ */
+public class EntityTable extends BaseTable<EntityTable> {
+ /** entity prefix */
+ private static final String PREFIX =
+ YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".entity";
+
+ /** config param name that specifies the entity table name */
+ public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
+
+ /**
+ * config param name that specifies the TTL for metrics column family in
+ * entity table
+ */
+ private static final String METRICS_TTL_CONF_NAME = PREFIX
+ + ".table.metrics.ttl";
+
+ /** default value for entity table name */
+ private static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
+
+ /** default TTL is 30 days for metrics timeseries */
+ private static final int DEFAULT_METRICS_TTL = 2592000;
+
+ /** default max number of versions */
+ private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
+
+ private static final Log LOG = LogFactory.getLog(EntityTable.class);
+
+ public EntityTable() {
+ super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.hadoop.yarn.server.timelineservice.storage.BaseTable#createTable
+ * (org.apache.hadoop.hbase.client.Admin,
+ * org.apache.hadoop.conf.Configuration)
+ */
+ public void createTable(Admin admin, Configuration hbaseConf)
+ throws IOException {
+
+ TableName table = getTableName(hbaseConf);
+ if (admin.tableExists(table)) {
+ // do not disable / delete existing table
+ // similar to the approach taken by map-reduce jobs when
+ // output directory exists
+ throw new IOException("Table " + table.getNameAsString()
+ + " already exists.");
+ }
+
+ HTableDescriptor entityTableDescp = new HTableDescriptor(table);
+ HColumnDescriptor infoCF =
+ new HColumnDescriptor(EntityColumnFamily.INFO.getBytes());
+ infoCF.setBloomFilterType(BloomType.ROWCOL);
+ entityTableDescp.addFamily(infoCF);
+
+ HColumnDescriptor configCF =
+ new HColumnDescriptor(EntityColumnFamily.CONFIGS.getBytes());
+ configCF.setBloomFilterType(BloomType.ROWCOL);
+ configCF.setBlockCacheEnabled(true);
+ entityTableDescp.addFamily(configCF);
+
+ HColumnDescriptor metricsCF =
+ new HColumnDescriptor(EntityColumnFamily.METRICS.getBytes());
+ entityTableDescp.addFamily(metricsCF);
+ metricsCF.setBlockCacheEnabled(true);
+ // always keep 1 version (the latest)
+ metricsCF.setMinVersions(1);
+ metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
+ metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
+ DEFAULT_METRICS_TTL));
+ entityTableDescp
+ .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
+ TimelineEntitySchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
+ admin.createTable(entityTableDescp,
+ TimelineEntitySchemaConstants.getUsernameSplits());
+ LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ + admin.tableExists(table));
+ }
+
+ /**
+ * @param metricsTTL time to live parameter for the metricss in this table.
+ * @param hbaseConf configururation in which to set the metrics TTL config
+ * variable.
+ */
+ public void setMetricsTTL(int metricsTTL, Configuration hbaseConf) {
+ hbaseConf.setInt(METRICS_TTL_CONF_NAME, metricsTTL);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
new file mode 100644
index 0000000..26f1cc5
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
index f999b4d..6abf240 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineWriterImpl.java
@@ -18,43 +18,41 @@
package org.apache.hadoop.yarn.server.timelineservice.storage;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.NavigableMap;
import java.util.Set;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.assertNotNull;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HBaseTestingUtility;
-import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.junit.BeforeClass;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
/**
- * Unit test HBaseTimelineWriterImpl
- * YARN 3411
- *
* @throws Exception
*/
public class TestHBaseTimelineWriterImpl {
@@ -69,12 +67,8 @@ public class TestHBaseTimelineWriterImpl {
}
private static void createSchema() throws IOException {
- byte[][] families = new byte[3][];
- families[0] = EntityColumnFamily.INFO.getInBytes();
- families[1] = EntityColumnFamily.CONFIG.getInBytes();
- families[2] = EntityColumnFamily.METRICS.getInBytes();
- TimelineSchemaCreator.createTimelineEntityTable(util.getHBaseAdmin(),
- util.getConfiguration());
+ new EntityTable()
+ .createTable(util.getHBaseAdmin(), util.getConfiguration());
}
@Test
@@ -151,18 +145,15 @@ public class TestHBaseTimelineWriterImpl {
// scan the table and see that entity exists
Scan s = new Scan();
- byte[] startRow = TimelineWriterUtils.getRowKeyPrefix(cluster, user, flow,
- runid, appName);
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
s.setStartRow(startRow);
s.setMaxVersions(Integer.MAX_VALUE);
- ResultScanner scanner = null;
- TableName entityTableName = TableName
- .valueOf(TimelineEntitySchemaConstants.DEFAULT_ENTITY_TABLE_NAME);
Connection conn = ConnectionFactory.createConnection(c1);
- Table entityTable = conn.getTable(entityTableName);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
int rowCount = 0;
int colCount = 0;
- scanner = entityTable.getScanner(s);
for (Result result : scanner) {
if (result != null && !result.isEmpty()) {
rowCount++;
@@ -172,37 +163,77 @@ public class TestHBaseTimelineWriterImpl {
entity));
// check info column family
- NavigableMap<byte[], byte[]> infoValues = result
- .getFamilyMap(EntityColumnFamily.INFO.getInBytes());
- String id1 = TimelineWriterUtils.getValueAsString(
- EntityColumnDetails.ID.getInBytes(), infoValues);
+ String id1 = EntityColumn.ID.readResult(result).toString();
assertEquals(id, id1);
- String type1 = TimelineWriterUtils.getValueAsString(
- EntityColumnDetails.TYPE.getInBytes(), infoValues);
+
+ String type1 = EntityColumn.TYPE.readResult(result).toString();
assertEquals(type, type1);
- Long cTime1 = TimelineWriterUtils.getValueAsLong(
- EntityColumnDetails.CREATED_TIME.getInBytes(), infoValues);
+
+ Number val = (Number) EntityColumn.CREATED_TIME.readResult(result);
+ Long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
- Long mTime1 = TimelineWriterUtils.getValueAsLong(
- EntityColumnDetails.MODIFIED_TIME.getInBytes(), infoValues);
+
+ val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
+ Long mTime1 = val.longValue();
assertEquals(mTime1, mTime);
- checkRelatedEntities(isRelatedTo, infoValues,
- EntityColumnDetails.PREFIX_IS_RELATED_TO.getInBytes());
- checkRelatedEntities(relatesTo, infoValues,
- EntityColumnDetails.PREFIX_RELATES_TO.getInBytes());
-
- // check config column family
- NavigableMap<byte[], byte[]> configValuesResult = result
- .getFamilyMap(EntityColumnFamily.CONFIG.getInBytes());
- checkConfigs(configValuesResult, conf);
-
- NavigableMap<byte[], byte[]> metricsResult = result
- .getFamilyMap(EntityColumnFamily.METRICS.getInBytes());
- checkMetricsSizeAndKey(metricsResult, metrics);
- List<Cell> metricCells = result.getColumnCells(
- EntityColumnFamily.METRICS.getInBytes(),
- Bytes.toBytes(m1.getId()));
- checkMetricsTimeseries(metricCells, m1);
+
+ // Remember isRelatedTo is of type Map<String, Set<String>>
+ for (String isRelatedToKey : isRelatedTo.keySet()) {
+ Object isRelatedToValue =
+ EntityColumnPrefix.IS_RELATED_TO.readResult(result,
+ isRelatedToKey);
+ String compoundValue = isRelatedToValue.toString();
+ // id7?id9?id6
+ Set<String> isRelatedToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(isRelatedTo.get(isRelatedToKey).size(),
+ isRelatedToValues.size());
+ for (String v : isRelatedTo.get(isRelatedToKey)) {
+ assertTrue(isRelatedToValues.contains(v));
+ }
+ }
+
+ // RelatesTo
+ for (String relatesToKey : relatesTo.keySet()) {
+ String compoundValue =
+ EntityColumnPrefix.RELATES_TO.readResult(result, relatesToKey)
+ .toString();
+ // id3?id4?id5
+ Set<String> relatesToValues =
+ new HashSet<String>(
+ Separator.VALUES.splitEncoded(compoundValue));
+ assertEquals(relatesTo.get(relatesToKey).size(),
+ relatesToValues.size());
+ for (String v : relatesTo.get(relatesToKey)) {
+ assertTrue(relatesToValues.contains(v));
+ }
+ }
+
+ // Configuration
+ Map<String, Object> configColumns =
+ EntityColumnPrefix.CONFIG.readResults(result);
+ assertEquals(conf.size(), configColumns.size());
+ for (String configItem : conf.keySet()) {
+ assertEquals(conf.get(configItem), configColumns.get(configItem));
+ }
+
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ EntityColumnPrefix.METRIC.readTimeseriesResults(result);
+
+ NavigableMap<Long, Number> metricMap = metricsResult.get(m1.getId());
+ // We got metrics back
+ assertNotNull(metricMap);
+ // Same number of metrics as we wrote
+ assertEquals(metricValues.entrySet().size(), metricMap.entrySet()
+ .size());
+
+ // Iterate over original metrics and confirm that they are present
+ // here.
+ for (Entry<Long, Number> metricEntry : metricValues.entrySet()) {
+ assertEquals(metricEntry.getValue(),
+ metricMap.get(metricEntry.getKey()));
+ }
}
}
assertEquals(1, rowCount);
@@ -212,80 +243,77 @@ public class TestHBaseTimelineWriterImpl {
hbi.stop();
hbi.close();
}
- }
-
- private void checkMetricsTimeseries(List<Cell> metricCells,
- TimelineMetric m1) throws IOException {
- Map<Long, Number> timeseries = m1.getValues();
- assertEquals(timeseries.size(), metricCells.size());
- for (Cell c1 : metricCells) {
- assertTrue(timeseries.containsKey(c1.getTimestamp()));
- assertEquals(GenericObjectMapper.read(CellUtil.cloneValue(c1)),
- timeseries.get(c1.getTimestamp()));
- }
- }
-
- private void checkMetricsSizeAndKey(
- NavigableMap<byte[], byte[]> metricsResult, Set<TimelineMetric> metrics) {
- assertEquals(metrics.size(), metricsResult.size());
- for (TimelineMetric m1 : metrics) {
- byte[] key = Bytes.toBytes(m1.getId());
- assertTrue(metricsResult.containsKey(key));
- }
- }
-
- private void checkConfigs(NavigableMap<byte[], byte[]> configValuesResult,
- Map<String, String> conf) throws IOException {
-
- assertEquals(conf.size(), configValuesResult.size());
- byte[] columnName;
- for (String key : conf.keySet()) {
- columnName = Bytes.toBytes(key);
- assertTrue(configValuesResult.containsKey(columnName));
- byte[] value = configValuesResult.get(columnName);
- assertNotNull(value);
- assertEquals(conf.get(key), GenericObjectMapper.read(value));
- }
- }
- private void checkRelatedEntities(Map<String, Set<String>> isRelatedTo,
- NavigableMap<byte[], byte[]> infoValues, byte[] columnPrefix)
- throws IOException {
-
- for (String key : isRelatedTo.keySet()) {
- byte[] columnName = TimelineWriterUtils.join(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES, columnPrefix,
- Bytes.toBytes(key));
-
- byte[] value = infoValues.get(columnName);
- assertNotNull(value);
- String isRelatedToEntities = GenericObjectMapper.read(value).toString();
- assertNotNull(isRelatedToEntities);
- assertEquals(
- TimelineWriterUtils.getValueAsString(
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR,
- isRelatedTo.get(key)), isRelatedToEntities);
- }
+ // Somewhat of a hack, not a separate test in order not to have to deal with
+ // test case order exectution.
+ testAdditionalEntity();
}
private boolean isRowKeyCorrect(byte[] rowKey, String cluster, String user,
String flow, Long runid, String appName, TimelineEntity te) {
- byte[][] rowKeyComponents = TimelineWriterUtils.split(rowKey,
- TimelineEntitySchemaConstants.ROW_KEY_SEPARATOR_BYTES);
+ byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey, -1);
assertTrue(rowKeyComponents.length == 7);
assertEquals(user, Bytes.toString(rowKeyComponents[0]));
assertEquals(cluster, Bytes.toString(rowKeyComponents[1]));
assertEquals(flow, Bytes.toString(rowKeyComponents[2]));
- assertEquals(TimelineWriterUtils.encodeRunId(runid),
- Bytes.toLong(rowKeyComponents[3]));
- assertEquals(TimelineWriterUtils.cleanse(appName), Bytes.toString(rowKeyComponents[4]));
+ assertEquals(EntityRowKey.invert(runid), Bytes.toLong(rowKeyComponents[3]));
+ assertEquals(appName, Bytes.toString(rowKeyComponents[4]));
assertEquals(te.getType(), Bytes.toString(rowKeyComponents[5]));
assertEquals(te.getId(), Bytes.toString(rowKeyComponents[6]));
return true;
}
+ private void testAdditionalEntity() throws IOException {
+ TimelineEvent event = new TimelineEvent();
+ event.setId("foo_event_id");
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId("attempt_1329348432655_0001_m_000008_18");
+ entity.setType("FOO_ATTEMPT");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+
+ HBaseTimelineWriterImpl hbi = null;
+ try {
+ Configuration c1 = util.getConfiguration();
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ String cluster = "cluster2";
+ String user = "user2";
+ String flow = "other_flow_name";
+ String flowVersion = "1111F01C2287BA";
+ long runid = 1009876543218L;
+ String appName = "some app name";
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, entities);
+ hbi.stop();
+ // scan the table and see that entity exists
+ Scan s = new Scan();
+ byte[] startRow =
+ EntityRowKey.getRowKeyPrefix(cluster, user, flow, runid, appName);
+ s.setStartRow(startRow);
+ s.setMaxVersions(Integer.MAX_VALUE);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ ResultScanner scanner = new EntityTable().getResultScanner(c1, conn, s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ if (result != null && !result.isEmpty()) {
+ rowCount++;
+ }
+ }
+ assertEquals(1, rowCount);
+
+ } finally {
+ hbi.stop();
+ hbi.close();
+ }
+ }
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
new file mode 100644
index 0000000..8b25a83
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestSeparator.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.Test;
+
+import com.google.common.collect.Iterables;
+
+public class TestSeparator {
+
+ private static String villain = "Dr. Heinz Doofenshmirtz";
+ private static String special =
+ ". * | ? + ( ) [ ] { } ^ $ \\ \"";
+
+ /**
+ *
+ */
+ @Test
+ public void testEncodeDecodeString() {
+
+ for (Separator separator : Separator.values()) {
+ testEncodeDecode(separator, "");
+ testEncodeDecode(separator, " ");
+ testEncodeDecode(separator, "!");
+ testEncodeDecode(separator, "?");
+ testEncodeDecode(separator, "&");
+ testEncodeDecode(separator, "+");
+ testEncodeDecode(separator, "Dr.");
+ testEncodeDecode(separator, "Heinz");
+ testEncodeDecode(separator, "Doofenshmirtz");
+ testEncodeDecode(separator, villain);
+ testEncodeDecode(separator, special);
+
+ assertNull(separator.encode(null));
+
+ }
+ }
+
+ private void testEncodeDecode(Separator separator, String token) {
+ String encoded = separator.encode(token);
+ String decoded = separator.decode(encoded);
+ String msg = "token:" + token + " separator:" + separator + ".";
+ assertEquals(msg, token, decoded);
+ }
+
+ @Test
+ public void testEncodeDecode() {
+ testEncodeDecode("Dr.", Separator.QUALIFIERS);
+ testEncodeDecode("Heinz", Separator.QUALIFIERS, Separator.QUALIFIERS);
+ testEncodeDecode("Doofenshmirtz", Separator.QUALIFIERS, null,
+ Separator.QUALIFIERS);
+ testEncodeDecode("&Perry", Separator.QUALIFIERS, Separator.VALUES, null);
+ testEncodeDecode("the ", Separator.QUALIFIERS, Separator.SPACE);
+ testEncodeDecode("Platypus...", (Separator) null);
+ testEncodeDecode("The what now ?!?", Separator.QUALIFIERS,
+ Separator.VALUES, Separator.SPACE);
+
+ }
+
+ /**
+ * Simple test to encode and decode using the same separators and confirm that
+ * we end up with the same as what we started with.
+ *
+ * @param token
+ * @param separators
+ */
+ private static void testEncodeDecode(String token, Separator... separators) {
+ byte[] encoded = Separator.encode(token, separators);
+ String decoded = Separator.decode(encoded, separators);
+ assertEquals(token, decoded);
+ }
+
+ @Test
+ public void testJoinStripped() {
+ List<String> stringList = new ArrayList<String>(0);
+ stringList.add("nothing");
+
+ String joined = Separator.VALUES.joinEncoded(stringList);
+ Iterable<String> split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ stringList = new ArrayList<String>(3);
+ stringList.add("a");
+ stringList.add("b?");
+ stringList.add("c");
+
+ joined = Separator.VALUES.joinEncoded(stringList);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(stringList, split));
+
+ String[] stringArray1 = { "else" };
+ joined = Separator.VALUES.joinEncoded(stringArray1);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray1), split));
+
+ String[] stringArray2 = { "d", "e?", "f" };
+ joined = Separator.VALUES.joinEncoded(stringArray2);
+ split = Separator.VALUES.splitEncoded(joined);
+ assertTrue(Iterables.elementsEqual(Arrays.asList(stringArray2), split));
+
+ List<String> empty = new ArrayList<String>(0);
+ split = Separator.VALUES.splitEncoded(null);
+ assertTrue(Iterables.elementsEqual(empty, split));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ed0625/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
new file mode 100644
index 0000000..4f96f87
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TestTimelineWriterUtils.java
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.junit.Test;
+
+public class TestTimelineWriterUtils {
+
+ @Test
+ public void test() {
+ // TODO: implement a test for the remaining method in TimelineWriterUtils.
+ }
+
+}