You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by gt...@apache.org on 2016/05/04 23:45:14 UTC
[01/50] [abbrv] hadoop git commit: YARN-4356. Ensure the timeline
service v.2 is disabled cleanly and has no impact when it's turned off.
Contributed by Sangjin Lee. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 bdd5d7dfa -> d2c4237fa (forced update)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
index 38b3172f..f31a98c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.DrainDispatcher;
@@ -535,7 +534,7 @@ public class TestApplication {
this.appId = BuilderUtils.newApplicationId(timestamp, id);
app = new ApplicationImpl(
- dispatcher, this.user, null, null, 0, appId, null, context);
+ dispatcher, this.user, appId, null, context);
containers = new ArrayList<Container>();
for (int i = 0; i < numContainers; i++) {
Container container = createMockedContainer(this.appId, i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
index e5ae1f8..c2dadd6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
@@ -31,17 +31,14 @@ import javax.ws.rs.core.MediaType;
import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.util.NodeHealthScriptRunner;
import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -63,6 +60,7 @@ import org.apache.hadoop.yarn.webapp.WebServicesTestUtils;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.w3c.dom.Document;
@@ -326,7 +324,7 @@ public class TestNMWebServices extends JerseyTestBase {
final String filename = "logfile1";
final String logMessage = "log message\n";
nmContext.getApplications().put(appId, new ApplicationImpl(null, "user",
- null, null, 0, appId, null, nmContext));
+ appId, null, nmContext));
MockContainer container = new MockContainer(appAttemptId,
new AsyncDispatcher(), new Configuration(), "user", appId, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 9cb9335..ac094fa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -308,8 +308,11 @@ public class ApplicationMasterService extends AbstractService implements
RMApp rmApp =
rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
+
// Remove collector address when app get finished.
- rmApp.removeCollectorAddr();
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ rmApp.removeCollectorAddr();
+ }
// checking whether the app exits in RMStateStore at first not to throw
// ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart.
@@ -577,8 +580,10 @@ public class ApplicationMasterService extends AbstractService implements
allocateResponse.setNumClusterNodes(this.rScheduler.getNumClusterNodes());
// add collector address for this application
- allocateResponse.setCollectorAddr(
- this.rmContext.getRMApps().get(applicationId).getCollectorAddr());
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ allocateResponse.setCollectorAddr(
+ this.rmContext.getRMApps().get(applicationId).getCollectorAddr());
+ }
// add preemption to the allocateResponse message (if any)
allocateResponse
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 1d74d35..577605f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -583,24 +583,27 @@ public class ClientRMService extends AbstractService implements
throw RPCUtil.getRemoteException(ie);
}
- // Sanity check for flow run
- String value = null;
- try {
- for (String tag : submissionContext.getApplicationTags()) {
- if (tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX + ":") ||
- tag.startsWith(
- TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) {
- value = tag.substring(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length() + 1);
- Long.valueOf(value);
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ // Sanity check for flow run
+ String value = null;
+ try {
+ for (String tag : submissionContext.getApplicationTags()) {
+ if (tag.startsWith(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX + ":") ||
+ tag.startsWith(
+ TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.toLowerCase() + ":")) {
+ value = tag.substring(TimelineUtils.FLOW_RUN_ID_TAG_PREFIX.length()
+ + 1);
+ Long.valueOf(value);
+ }
}
+ } catch (NumberFormatException e) {
+ LOG.warn("Invalid to flow run: " + value +
+ ". Flow run should be a long integer", e);
+ RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
+ e.getMessage(), "ClientRMService",
+ "Exception in submitting application", applicationId);
+ throw RPCUtil.getRemoteException(e);
}
- } catch (NumberFormatException e) {
- LOG.warn("Invalid to flow run: " + value +
- ". Flow run should be a long integer", e);
- RMAuditLogger.logFailure(user, AuditConstants.SUBMIT_APP_REQUEST,
- e.getMessage(), "ClientRMService",
- "Exception in submitting application", applicationId);
- throw RPCUtil.getRemoteException(e);
}
// Check whether app has already been put into rmContext,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 55029ff..30efc8e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -399,8 +399,11 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
LOG.warn(message);
throw new YarnException(message);
}
- // Start timeline collector for the submitted app
- application.startTimelineCollector();
+
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ // Start timeline collector for the submitted app
+ application.startTimelineCollector();
+ }
// Inform the ACLs Manager
this.applicationACLsManager.addApplication(applicationId,
submissionContext.getAMContainerSpec().getApplicationACLs());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 8f6be41..6b03de0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -469,18 +469,19 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
protected SystemMetricsPublisher createSystemMetricsPublisher() {
- boolean timelineServiceEnabled =
- conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
- SystemMetricsPublisher publisher = null;
- if (timelineServiceEnabled) {
- if (conf.getBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
- YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED)) {
- LOG.info("TimelineService V1 is configured");
- publisher = new TimelineServiceV1Publisher();
- } else {
- LOG.info("TimelineService V2 is configured");
+ SystemMetricsPublisher publisher;
+ if (YarnConfiguration.timelineServiceEnabled(conf) &&
+ YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ // we're dealing with the v.2.x publisher
+ LOG.info("system metrics publisher with the timeline service V2 is " +
+ "configured");
publisher = new TimelineServiceV2Publisher(rmContext);
+ } else {
+ // we're dealing with the v.1.x publisher
+ LOG.info("system metrics publisher with the timeline service V1 is " +
+ "configured");
+ publisher = new TimelineServiceV1Publisher();
}
} else {
LOG.info("TimelineServicePublisher is not configured");
@@ -606,10 +607,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
addService(rmApplicationHistoryWriter);
rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
- RMTimelineCollectorManager timelineCollectorManager =
- createRMTimelineCollectorManager();
- addService(timelineCollectorManager);
- rmContext.setRMTimelineCollectorManager(timelineCollectorManager);
+ if (YarnConfiguration.timelineServiceV2Enabled(configuration)) {
+ RMTimelineCollectorManager timelineCollectorManager =
+ createRMTimelineCollectorManager();
+ addService(timelineCollectorManager);
+ rmContext.setRMTimelineCollectorManager(timelineCollectorManager);
+ }
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 909ab74..1dbbeb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -498,10 +498,15 @@ public class ResourceTrackerService extends AbstractService implements
message);
}
- // Check & update collectors info from request.
- // TODO make sure it won't have race condition issue for AM failed over case
- // that the older registration could possible override the newer one.
- updateAppCollectorsMap(request);
+ boolean timelineV2Enabled =
+ YarnConfiguration.timelineServiceV2Enabled(getConfig());
+ if (timelineV2Enabled) {
+ // Check & update collectors info from request.
+ // TODO make sure it won't have race condition issue for AM failed over
+ // case that the older registration could possible override the newer
+ // one.
+ updateAppCollectorsMap(request);
+ }
// Heartbeat response
NodeHeartbeatResponse nodeHeartBeatResponse = YarnServerBuilderUtils
@@ -520,12 +525,12 @@ public class ResourceTrackerService extends AbstractService implements
nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
}
- // Return collectors' map that NM needs to know
- // TODO we should optimize this to only include collector info that NM
- // doesn't know yet.
List<ApplicationId> keepAliveApps =
remoteNodeStatus.getKeepAliveApplications();
- if (keepAliveApps != null) {
+ if (timelineV2Enabled && keepAliveApps != null) {
+ // Return collectors' map that NM needs to know
+ // TODO we should optimize this to only include collector info that NM
+ // doesn't know yet.
setAppCollectorsMapToResponse(keepAliveApps, nodeHeartBeatResponse);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index df760a3..9a7638c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -51,6 +51,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.client.NMProxy;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -215,12 +216,14 @@ public class AMLauncher implements Runnable {
.get(applicationId)
.getSubmitTime()));
- // Set flow context info
- for (String tag :
- rmContext.getRMApps().get(applicationId).getApplicationTags()) {
- setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, tag);
- setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, tag);
- setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, tag);
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ // Set flow context info
+ for (String tag :
+ rmContext.getRMApps().get(applicationId).getApplicationTags()) {
+ setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, tag);
+ setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, tag);
+ setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, tag);
+ }
}
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 0105495..b96114e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -58,7 +58,7 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
/**
- * This class is responsible for posting application, appattempt & Container
+ * This class is responsible for posting application, appattempt & Container
* lifecycle related events to timeline service V2
*/
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 6f6a716..ca2771a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -971,15 +971,17 @@ public class RMAppImpl implements RMApp, Recoverable {
extends RMAppTransition {
public void transition(RMAppImpl app, RMAppEvent event) {
- LOG.info("Updating collector info for app: " + app.getApplicationId());
+ if (YarnConfiguration.timelineServiceV2Enabled(app.conf)) {
+ LOG.info("Updating collector info for app: " + app.getApplicationId());
- RMAppCollectorUpdateEvent appCollectorUpdateEvent =
- (RMAppCollectorUpdateEvent) event;
- // Update collector address
- app.setCollectorAddr(appCollectorUpdateEvent.getAppCollectorAddr());
+ RMAppCollectorUpdateEvent appCollectorUpdateEvent =
+ (RMAppCollectorUpdateEvent) event;
+ // Update collector address
+ app.setCollectorAddr(appCollectorUpdateEvent.getAppCollectorAddr());
- // TODO persistent to RMStateStore for recover
- // Save to RMStateStore
+ // TODO persistent to RMStateStore for recover
+ // Save to RMStateStore
+ }
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 9261192..5d70dd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -652,6 +652,7 @@ public class TestClientRMService {
ClientRMService rmService =
new ClientRMService(rmContext, yarnScheduler, appManager,
mockAclsManager, mockQueueACLsManager, null);
+ rmService.init(new Configuration());
// without name and queue
@@ -744,6 +745,7 @@ public class TestClientRMService {
ClientRMService rmService =
new ClientRMService(rmContext, yarnScheduler, appManager,
mockAclsManager, mockQueueACLsManager, null);
+ rmService.init(new Configuration());
// Initialize appnames and queues
String[] queues = {QUEUE_1, QUEUE_2};
@@ -907,6 +909,7 @@ public class TestClientRMService {
final ClientRMService rmService =
new ClientRMService(rmContext, yarnScheduler, appManager, null, null,
null);
+ rmService.init(new Configuration());
// submit an app and wait for it to block while in app submission
Thread t = new Thread() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 2652acf..78435b5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -75,7 +75,7 @@ public class TestSystemMetricsPublisher {
public static void setup() throws Exception {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
- conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STORE,
MemoryTimelineStore.class, TimelineStore.class);
conf.setClass(YarnConfiguration.TIMELINE_SERVICE_STATE_STORE_CLASS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 20a5b13..baaa566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -131,6 +131,7 @@ public class TestSystemMetricsPublisherForV2 {
private static Configuration getTimelineV2Conf() {
Configuration conf = new Configuration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setInt(
YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, 2);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
index 5672759..a734340 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/timelineservice/TestTimelineServiceClientIntegration.java
@@ -20,41 +20,55 @@ package org.apache.hadoop.yarn.server.timelineservice;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.any;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.timelineservice.*;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationAttemptEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ClusterEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.QueueEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.api.records.timelineservice.UserEntity;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.CollectorNodemanagerProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorContextResponse;
-import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.collector.NodeTimelineCollectorManager;
+import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-import java.io.IOException;
-
public class TestTimelineServiceClientIntegration {
private static NodeTimelineCollectorManager collectorManager;
private static PerNodeTimelineCollectorsAuxService auxService;
+ private static Configuration conf;
@BeforeClass
public static void setupClass() throws Exception {
try {
collectorManager = new MockNodeTimelineCollectorManager();
+ conf = new YarnConfiguration();
+ // enable timeline service v.2
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
- collectorManager);
+ collectorManager, conf);
auxService.addApplication(ApplicationId.newInstance(0, 1));
} catch (ExitUtil.ExitException e) {
fail();
@@ -76,7 +90,7 @@ public class TestTimelineServiceClientIntegration {
// set the timeline service address manually
client.setTimelineServiceAddress(
collectorManager.getRestServerBindAddress());
- client.init(new YarnConfiguration());
+ client.init(conf);
client.start();
TimelineEntity entity = new TimelineEntity();
entity.setType("test entity type");
@@ -103,7 +117,7 @@ public class TestTimelineServiceClientIntegration {
// set the timeline service address manually
client.setTimelineServiceAddress(
collectorManager.getRestServerBindAddress());
- client.init(new YarnConfiguration());
+ client.init(conf);
client.start();
ClusterEntity cluster = new ClusterEntity();
cluster.setId(YarnConfiguration.DEFAULT_RM_CLUSTER_ID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 4147d42..0319e34 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -30,12 +30,11 @@ import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.api.ApplicationInitializationContext;
import org.apache.hadoop.yarn.server.api.ApplicationTerminationContext;
import org.apache.hadoop.yarn.server.api.AuxiliaryService;
-import org.apache.hadoop.yarn.server.api.ContainerContext;
import org.apache.hadoop.yarn.server.api.ContainerInitializationContext;
import org.apache.hadoop.yarn.server.api.ContainerTerminationContext;
import org.apache.hadoop.yarn.server.api.ContainerType;
@@ -68,6 +67,9 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
+ if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ throw new YarnException("Timeline service v2 is not enabled");
+ }
collectorManager.init(conf);
super.serviceInit(conf);
}
@@ -175,7 +177,8 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
@VisibleForTesting
public static PerNodeTimelineCollectorsAuxService
- launchServer(String[] args, NodeTimelineCollectorManager collectorManager) {
+ launchServer(String[] args, NodeTimelineCollectorManager collectorManager,
+ Configuration conf) {
Thread
.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(
@@ -187,7 +190,6 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
new PerNodeTimelineCollectorsAuxService(collectorManager);
ShutdownHookManager.get().addShutdownHook(new ShutdownHook(auxService),
SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
auxService.init(conf);
auxService.start();
} catch (Throwable t) {
@@ -210,6 +212,9 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
}
public static void main(String[] args) {
- launchServer(args, null);
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ launchServer(args, null, conf);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index afe1536..6b4213d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.util.ShutdownHookManager;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
@@ -66,6 +67,10 @@ public class TimelineReaderServer extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
+ if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ throw new YarnException("timeline service v.2 is not enabled");
+ }
+
TimelineReader timelineReaderStore = createTimelineReaderStore(conf);
addService(timelineReaderStore);
timelineReaderManager = createTimelineReaderManager(timelineReaderStore);
@@ -143,7 +148,8 @@ public class TimelineReaderServer extends CompositeService {
return readerWebServer.getConnectorAddress(0).getPort();
}
- static TimelineReaderServer startTimelineReaderServer(String[] args) {
+ static TimelineReaderServer startTimelineReaderServer(String[] args,
+ Configuration conf) {
Thread.setDefaultUncaughtExceptionHandler(
new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(TimelineReaderServer.class,
@@ -154,7 +160,6 @@ public class TimelineReaderServer extends CompositeService {
ShutdownHookManager.get().addShutdownHook(
new CompositeServiceShutdownHook(timelineReaderServer),
SHUTDOWN_HOOK_PRIORITY);
- YarnConfiguration conf = new YarnConfiguration();
timelineReaderServer.init(conf);
timelineReaderServer.start();
} catch (Throwable t) {
@@ -165,6 +170,9 @@ public class TimelineReaderServer extends CompositeService {
}
public static void main(String[] args) {
- startTimelineReaderServer(args);
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ startTimelineReaderServer(args, conf);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index dafc76e..4fdf47e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -50,11 +50,16 @@ import java.io.IOException;
public class TestPerNodeTimelineCollectorsAuxService {
private ApplicationAttemptId appAttemptId;
private PerNodeTimelineCollectorsAuxService auxService;
+ private Configuration conf;
public TestPerNodeTimelineCollectorsAuxService() {
ApplicationId appId =
ApplicationId.newInstance(System.currentTimeMillis(), 1);
appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ conf = new YarnConfiguration();
+ // enable timeline service v.2
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
}
@After
@@ -134,7 +139,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
try {
auxService =
PerNodeTimelineCollectorsAuxService.launchServer(new String[0],
- createCollectorManager());
+ createCollectorManager(), conf);
} catch (ExitUtil.ExitException e) {
assertEquals(0, e.status);
ExitUtil.resetFirstExitException();
@@ -160,7 +165,7 @@ public class TestPerNodeTimelineCollectorsAuxService {
NodeTimelineCollectorManager collectorManager = createCollectorManager();
PerNodeTimelineCollectorsAuxService auxService =
spy(new PerNodeTimelineCollectorsAuxService(collectorManager));
- auxService.init(new YarnConfiguration());
+ auxService.init(conf);
auxService.start();
return auxService;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
index 7098814..b42488c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderServer.java
@@ -30,8 +30,11 @@ public class TestTimelineReaderServer {
@Test(timeout = 60000)
public void testStartStopServer() throws Exception {
+ @SuppressWarnings("resource")
TimelineReaderServer server = new TimelineReaderServer();
Configuration config = new YarnConfiguration();
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
"localhost:0");
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 45bce2f..91f6ee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -72,6 +72,8 @@ public class TestTimelineReaderWebServices {
public void init() throws Exception {
try {
Configuration config = new YarnConfiguration();
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
"localhost:0");
config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 3b285aa..818cd89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -238,6 +238,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
public void init() throws Exception {
try {
Configuration config = util.getConfiguration();
+ config.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ config.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
config.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
"localhost:0");
config.set(YarnConfiguration.RM_CLUSTER_ID, "cluster1");
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: YARN-4622. TestDistributedShell
fails for v2 test cases after modifications for 1.5. (Naganarasimha G R via
Varun Saxena)
Posted by gt...@apache.org.
YARN-4622. TestDistributedShell fails for v2 test cases after modifications for 1.5. (Naganarasimha G R via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2269e78a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2269e78a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2269e78a
Branch: refs/heads/YARN-2928
Commit: 2269e78adbd3969e84b83a541d2cd84498c46f47
Parents: 2d1fc87
Author: Varun Saxena <va...@apache.org>
Authored: Fri Jan 22 01:41:25 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:24:10 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/client/api/impl/TimelineClientImpl.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2269e78a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index f36167d..4db3b71 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.map.ObjectMapper;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
@@ -334,9 +335,10 @@ public class TimelineClientImpl extends TimelineClient {
@Override
protected void serviceStart() throws Exception {
- timelineWriter = createTimelineWriter(
- configuration, authUgi, client,
- constructResURI(getConfig(), timelineServiceAddress, false));
+ if (!timelineServiceV2) {
+ timelineWriter = createTimelineWriter(configuration, authUgi, client,
+ constructResURI(getConfig(), timelineServiceAddress, false));
+ }
}
protected TimelineWriter createTimelineWriter(Configuration conf,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: YARN-3367. Replace starting a
separate thread for post entity with event loop in TimelineClient
(Naganarasimha G R via sjlee)
Posted by gt...@apache.org.
YARN-3367. Replace starting a separate thread for post entity with event loop in TimelineClient (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f59c7433
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f59c7433
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f59c7433
Branch: refs/heads/YARN-2928
Commit: f59c74338313305752678ea140fa3195d9fee07e
Parents: ee803e1
Author: Sangjin Lee <sj...@apache.org>
Authored: Tue Feb 9 09:07:37 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:33:32 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 81 +----
.../mapred/JobHistoryFileReplayMapper.java | 8 +-
.../hadoop/mapred/TimelineEntityConverter.java | 12 +-
.../timelineservice/TimelineEntities.java | 17 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 6 +
.../distributedshell/ApplicationMaster.java | 78 +----
.../api/async/impl/AMRMClientAsyncImpl.java | 26 +-
.../hadoop/yarn/client/api/TimelineClient.java | 8 +-
.../client/api/impl/TimelineClientImpl.java | 285 ++++++++++++++---
.../src/main/resources/yarn-default.xml | 7 +
.../api/impl/TestTimelineClientV2Impl.java | 304 +++++++++++++++++++
.../nodemanager/NodeStatusUpdaterImpl.java | 4 +-
12 files changed, 619 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 853c506..639f56e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -28,10 +28,7 @@ import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -85,7 +82,6 @@ import org.codehaus.jackson.node.ObjectNode;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientHandlerException;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* The job history events get routed to this class. This class writes the Job
* history events to the DFS directly into a staging dir and then moved to a
@@ -141,10 +137,6 @@ public class JobHistoryEventHandler extends AbstractService
private boolean timelineServiceV2Enabled = false;
- // For posting entities in new timeline service in a non-blocking way
- // TODO YARN-3367 replace with event loop in TimelineClient.
- private ExecutorService threadPool;
-
private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB";
private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK";
private static final String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE =
@@ -284,10 +276,6 @@ public class JobHistoryEventHandler extends AbstractService
YarnConfiguration.timelineServiceV2Enabled(conf);
LOG.info("Timeline service is enabled; version: " +
YarnConfiguration.getTimelineServiceVersion(conf));
- if (timelineServiceV2Enabled) {
- // initialize the thread pool for v.2 timeline service
- threadPool = createThreadPool();
- }
} else {
LOG.info("Timeline service is not enabled");
}
@@ -461,35 +449,9 @@ public class JobHistoryEventHandler extends AbstractService
if (timelineClient != null) {
timelineClient.stop();
}
- if (threadPool != null) {
- shutdownAndAwaitTermination();
- }
LOG.info("Stopped JobHistoryEventHandler. super.stop()");
super.serviceStop();
}
-
- // TODO remove threadPool after adding non-blocking call in TimelineClient
- private ExecutorService createThreadPool() {
- return Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
- }
-
- private void shutdownAndAwaitTermination() {
- threadPool.shutdown();
- try {
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
- threadPool.shutdownNow();
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
- LOG.error("ThreadPool did not terminate");
- }
- }
- } catch (InterruptedException ie) {
- threadPool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
protected EventWriter createEventWriter(Path historyFilePath)
throws IOException {
@@ -1097,41 +1059,6 @@ public class JobHistoryEventHandler extends AbstractService
+ "Server", ex);
}
}
-
- @Private
- public JsonNode countersToJSON(Counters counters) {
- ArrayNode nodes = FACTORY.arrayNode();
- if (counters != null) {
- for (CounterGroup counterGroup : counters) {
- ObjectNode groupNode = nodes.addObject();
- groupNode.put("NAME", counterGroup.getName());
- groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
- ArrayNode countersNode = groupNode.putArray("COUNTERS");
- for (Counter counter : counterGroup) {
- ObjectNode counterNode = countersNode.addObject();
- counterNode.put("NAME", counter.getName());
- counterNode.put("DISPLAY_NAME", counter.getDisplayName());
- counterNode.put("VALUE", counter.getValue());
- }
- }
- }
- return nodes;
- }
-
- private void putEntityWithoutBlocking(final TimelineClient client,
- final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity) {
- Runnable publishWrapper = new Runnable() {
- public void run() {
- try {
- client.putEntities(entity);
- } catch (IOException|YarnException e) {
- LOG.error("putEntityNonBlocking get failed: " + e);
- throw new RuntimeException(e.toString());
- }
- }
- };
- threadPool.execute(publishWrapper);
- }
// create JobEntity from HistoryEvent with adding other info, like:
// jobId, timestamp and entityType.
@@ -1293,7 +1220,13 @@ public class JobHistoryEventHandler extends AbstractService
taskId, setCreatedTime);
}
}
- putEntityWithoutBlocking(timelineClient, tEntity);
+ try {
+ timelineClient.putEntitiesAsync(tEntity);
+ } catch (IOException | YarnException e) {
+ LOG.error("Failed to process Event " + event.getEventType()
+ + " for the job : " + jobId, e);
+ }
+
}
private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
index 802b78f..4fb5308 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.mapred;
import java.io.IOException;
import java.util.Collection;
import java.util.HashMap;
+import java.util.List;
import java.util.Map;
-import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -176,7 +176,7 @@ class JobHistoryFileReplayMapper extends EntityWriter {
// create entities from job history and write them
long totalTime = 0;
- Set<TimelineEntity> entitySet =
+ List<TimelineEntity> entitySet =
converter.createTimelineEntities(jobInfo, jobConf);
LOG.info("converted them into timeline entities for job " + jobIdStr);
// use the current user for this purpose
@@ -215,7 +215,7 @@ class JobHistoryFileReplayMapper extends EntityWriter {
}
private void writeAllEntities(AppLevelTimelineCollector collector,
- Set<TimelineEntity> entitySet, UserGroupInformation ugi)
+ List<TimelineEntity> entitySet, UserGroupInformation ugi)
throws IOException {
TimelineEntities entities = new TimelineEntities();
entities.setEntities(entitySet);
@@ -223,7 +223,7 @@ class JobHistoryFileReplayMapper extends EntityWriter {
}
private void writePerEntity(AppLevelTimelineCollector collector,
- Set<TimelineEntity> entitySet, UserGroupInformation ugi)
+ List<TimelineEntity> entitySet, UserGroupInformation ugi)
throws IOException {
for (TimelineEntity entity : entitySet) {
TimelineEntities entities = new TimelineEntities();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
index 880014b..0e2eb72 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
@@ -18,7 +18,9 @@
package org.apache.hadoop.mapred;
+import java.util.ArrayList;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -57,16 +59,16 @@ class TimelineEntityConverter {
* Note that we also do not add info to the YARN application entity, which
* would be needed for aggregation.
*/
- public Set<TimelineEntity> createTimelineEntities(JobInfo jobInfo,
+ public List<TimelineEntity> createTimelineEntities(JobInfo jobInfo,
Configuration conf) {
- Set<TimelineEntity> entities = new HashSet<>();
+ List<TimelineEntity> entities = new ArrayList<>();
// create the job entity
TimelineEntity job = createJobEntity(jobInfo, conf);
entities.add(job);
// create the task and task attempt entities
- Set<TimelineEntity> tasksAndAttempts =
+ List<TimelineEntity> tasksAndAttempts =
createTaskAndTaskAttemptEntities(jobInfo);
entities.addAll(tasksAndAttempts);
@@ -125,9 +127,9 @@ class TimelineEntityConverter {
}
}
- private Set<TimelineEntity> createTaskAndTaskAttemptEntities(
+ private List<TimelineEntity> createTaskAndTaskAttemptEntities(
JobInfo jobInfo) {
- Set<TimelineEntity> entities = new HashSet<>();
+ List<TimelineEntity> entities = new ArrayList<>();
Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
" tasks");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
index f08a0ec..63989e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
@@ -17,15 +17,16 @@
*/
package org.apache.hadoop.yarn.api.records.timelineservice;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
+import java.util.ArrayList;
+import java.util.List;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.HashSet;
-import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
/**
* This class hosts a set of timeline entities.
@@ -36,22 +37,22 @@ import java.util.Set;
@InterfaceStability.Unstable
public class TimelineEntities {
- private Set<TimelineEntity> entities = new HashSet<>();
+ private List<TimelineEntity> entities = new ArrayList<>();
public TimelineEntities() {
}
@XmlElement(name = "entities")
- public Set<TimelineEntity> getEntities() {
+ public List<TimelineEntity> getEntities() {
return entities;
}
- public void setEntities(Set<TimelineEntity> timelineEntities) {
+ public void setEntities(List<TimelineEntity> timelineEntities) {
this.entities = timelineEntities;
}
- public void addEntities(Set<TimelineEntity> timelineEntities) {
+ public void addEntities(List<TimelineEntity> timelineEntities) {
this.entities.addAll(timelineEntities);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 2490dda..5115758 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1905,6 +1905,12 @@ public class YarnConfiguration extends Configuration {
public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000;
+ public static final String NUMBER_OF_ASYNC_ENTITIES_TO_MERGE =
+ TIMELINE_SERVICE_PREFIX
+ + "timeline-client.number-of-async-entities-to-merge";
+
+ public static final int DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE = 10;
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 6bd3c26..dbe9c64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -40,9 +40,6 @@ import java.util.Set;
import java.util.Vector;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.cli.CommandLine;
@@ -111,7 +108,6 @@ import org.apache.log4j.LogManager;
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientHandlerException;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* An ApplicationMaster for executing shell commands on a set of launched
@@ -225,10 +221,6 @@ public class ApplicationMaster {
private boolean timelineServiceV2 = false;
- // For posting entities in new timeline service in a non-blocking way
- // TODO replace with event loop in TimelineClient.
- private ExecutorService threadPool;
-
// App Master configuration
// No. of containers to run shell command on
@VisibleForTesting
@@ -328,10 +320,6 @@ public class ApplicationMaster {
}
appMaster.run();
result = appMaster.finish();
-
- if (appMaster.threadPool != null) {
- appMaster.shutdownAndAwaitTermination();
- }
} catch (Throwable t) {
LOG.fatal("Error running ApplicationMaster", t);
LogManager.shutdown();
@@ -346,29 +334,6 @@ public class ApplicationMaster {
}
}
- //TODO remove threadPool after adding non-blocking call in TimelineClient
- private ExecutorService createThreadPool() {
- return Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
- }
-
- private void shutdownAndAwaitTermination() {
- threadPool.shutdown();
- try {
- // Wait a while for existing tasks to terminate
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
- threadPool.shutdownNow();
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
- LOG.error("ThreadPool did not terminate");
- }
- } catch (InterruptedException ie) {
- threadPool.shutdownNow();
- // Preserve interrupt status
- Thread.currentThread().interrupt();
- }
- }
-
/**
* Dump out contents of $CWD and the environment to stdout for debugging
*/
@@ -591,11 +556,7 @@ public class ApplicationMaster {
"container_retry_interval", "0"));
if (YarnConfiguration.timelineServiceEnabled(conf)) {
- timelineServiceV2 =
- YarnConfiguration.timelineServiceV2Enabled(conf);
- if (timelineServiceV2) {
- threadPool = createThreadPool();
- }
+ timelineServiceV2 = YarnConfiguration.timelineServiceV2Enabled(conf);
} else {
timelineClient = null;
LOG.warn("Timeline service is not enabled");
@@ -746,8 +707,10 @@ public class ApplicationMaster {
if (timelineServiceV2) {
timelineClient = TimelineClient.createTimelineClient(
appAttemptID.getApplicationId());
+ LOG.info("Timeline service V2 client is enabled");
} else {
timelineClient = TimelineClient.createTimelineClient();
+ LOG.info("Timeline service V1 client is enabled");
}
timelineClient.init(conf);
timelineClient.start();
@@ -1385,18 +1348,8 @@ public class ApplicationMaster {
shellId);
return new Thread(runnableLaunchContainer);
}
-
- private void publishContainerStartEventOnTimelineServiceV2(
- final Container container) {
- Runnable publishWrapper = new Runnable() {
- public void run() {
- publishContainerStartEventOnTimelineServiceV2Base(container);
- }
- };
- threadPool.execute(publishWrapper);
- }
- private void publishContainerStartEventOnTimelineServiceV2Base(
+ private void publishContainerStartEventOnTimelineServiceV2(
Container container) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
@@ -1430,16 +1383,6 @@ public class ApplicationMaster {
private void publishContainerEndEventOnTimelineServiceV2(
final ContainerStatus container) {
- Runnable publishWrapper = new Runnable() {
- public void run() {
- publishContainerEndEventOnTimelineServiceV2Base(container);
- }
- };
- threadPool.execute(publishWrapper);
- }
-
- private void publishContainerEndEventOnTimelineServiceV2Base(
- final ContainerStatus container) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getContainerId().toString());
@@ -1470,17 +1413,6 @@ public class ApplicationMaster {
}
private void publishApplicationAttemptEventOnTimelineServiceV2(
- final DSEvent appEvent) {
-
- Runnable publishWrapper = new Runnable() {
- public void run() {
- publishApplicationAttemptEventOnTimelineServiceV2Base(appEvent);
- }
- };
- threadPool.execute(publishWrapper);
- }
-
- private void publishApplicationAttemptEventOnTimelineServiceV2Base(
DSEvent appEvent) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
@@ -1498,7 +1430,7 @@ public class ApplicationMaster {
appSubmitterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public TimelinePutResponse run() throws Exception {
- timelineClient.putEntities(entity);
+ timelineClient.putEntitiesAsync(entity);
return null;
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
index 212f721..8af0c78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/async/impl/AMRMClientAsyncImpl.java
@@ -41,9 +41,9 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
import org.apache.hadoop.yarn.client.api.impl.AMRMClientImpl;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -327,6 +327,19 @@ extends AMRMClientAsync<T> {
LOG.info("Interrupted while waiting for queue", ex);
continue;
}
+
+ String collectorAddress = response.getCollectorAddr();
+ TimelineClient timelineClient = client.getRegisteredTimeineClient();
+ if (timelineClient != null && collectorAddress != null
+ && !collectorAddress.isEmpty()) {
+ if (collectorAddr == null
+ || !collectorAddr.equals(collectorAddress)) {
+ collectorAddr = collectorAddress;
+ timelineClient.setTimelineServiceAddress(collectorAddress);
+ LOG.info("collectorAddress " + collectorAddress);
+ }
+ }
+
List<NodeReport> updatedNodes = response.getUpdatedNodes();
if (!updatedNodes.isEmpty()) {
handler.onNodesUpdated(updatedNodes);
@@ -354,17 +367,6 @@ extends AMRMClientAsync<T> {
if (!allocated.isEmpty()) {
handler.onContainersAllocated(allocated);
}
-
- String collectorAddress = response.getCollectorAddr();
- TimelineClient timelineClient = client.getRegisteredTimeineClient();
- if (timelineClient != null && collectorAddress != null
- && !collectorAddress.isEmpty()) {
- if (collectorAddr == null ||
- !collectorAddr.equals(collectorAddress)) {
- collectorAddr = collectorAddress;
- timelineClient.setTimelineServiceAddress(collectorAddress);
- }
- }
progress = handler.getProgress();
} catch (Throwable ex) {
handler.onError(ex);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index 58307d4..e043c2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -28,12 +28,12 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineDomain;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timeline.TimelineEntityGroupId;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -55,7 +55,7 @@ public abstract class TimelineClient extends AbstractService implements
* construct and initialize a timeline client if the following operations are
* supposed to be conducted by that user.
*/
- private ApplicationId contextAppId;
+ protected ApplicationId contextAppId;
/**
* Creates an instance of the timeline v.1.x client.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 06b3ac4..87a5e9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -30,6 +30,14 @@ import java.net.URL;
import java.net.URLConnection;
import java.security.GeneralSecurityException;
import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.FutureTask;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.HttpsURLConnection;
@@ -130,6 +138,8 @@ public class TimelineClientImpl extends TimelineClient {
@VisibleForTesting
TimelineClientConnectionRetry connectionRetry;
+ private TimelineEntityDispatcher entityDispatcher;
+
// Abstract class for an operation that should be retried by timeline client
@Private
@VisibleForTesting
@@ -315,6 +325,7 @@ public class TimelineClientImpl extends TimelineClient {
serviceRetryInterval = conf.getLong(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
+ entityDispatcher = new TimelineEntityDispatcher(conf);
} else {
if (YarnConfiguration.useHttps(conf)) {
setTimelineServiceAddress(conf.get(
@@ -335,7 +346,9 @@ public class TimelineClientImpl extends TimelineClient {
@Override
protected void serviceStart() throws Exception {
- if (!timelineServiceV2) {
+ if (timelineServiceV2) {
+ entityDispatcher.start();
+ } else {
timelineWriter = createTimelineWriter(configuration, authUgi, client,
constructResURI(getConfig(), timelineServiceAddress, false));
}
@@ -357,6 +370,9 @@ public class TimelineClientImpl extends TimelineClient {
if (this.timelineWriter != null) {
this.timelineWriter.close();
}
+ if (timelineServiceV2) {
+ entityDispatcher.stop();
+ }
super.serviceStop();
}
@@ -376,37 +392,21 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public void putEntities(
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
- throws IOException, YarnException {
- putEntities(false, entities);
+ throws IOException, YarnException {
+ if (!timelineServiceV2) {
+ throw new YarnException("v.2 method is invoked on a v.1.x client");
+ }
+ entityDispatcher.dispatchEntities(true, entities);
}
@Override
public void putEntitiesAsync(
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
throws IOException, YarnException {
- putEntities(true, entities);
- }
-
- private void putEntities(boolean async,
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
- throws IOException, YarnException {
if (!timelineServiceV2) {
throw new YarnException("v.2 method is invoked on a v.1.x client");
}
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities
- entitiesContainer =
- new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities();
- for (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity : entities) {
- entitiesContainer.addEntity(entity);
- }
- MultivaluedMap<String, String> params = new MultivaluedMapImpl();
- if (getContextAppId() != null) {
- params.add("appid", getContextAppId().toString());
- }
- if (async) {
- params.add("async", Boolean.TRUE.toString());
- }
- putObjects("entities", params, entitiesContainer);
+ entityDispatcher.dispatchEntities(false, entities);
}
@Override
@@ -417,20 +417,10 @@ public class TimelineClientImpl extends TimelineClient {
// Used for new timeline service only
@Private
- public void putObjects(String path, MultivaluedMap<String, String> params,
+ protected void putObjects(String path, MultivaluedMap<String, String> params,
Object obj) throws IOException, YarnException {
- // timelineServiceAddress could haven't be initialized yet
- // or stale (only for new timeline service)
- int retries = pollTimelineServiceAddress(this.maxServiceRetries);
- if (timelineServiceAddress == null) {
- String errMessage = "TimelineClient has reached to max retry times : "
- + this.maxServiceRetries
- + ", but failed to fetch timeline service address. Please verify"
- + " Timeline Auxillary Service is configured in all the NMs";
- LOG.error(errMessage);
- throw new YarnException(errMessage);
- }
+ int retries = verifyRestEndPointAvailable();
// timelineServiceAddress could be stale, add retry logic here.
boolean needRetry = true;
@@ -448,6 +438,21 @@ public class TimelineClientImpl extends TimelineClient {
}
}
+ private int verifyRestEndPointAvailable() throws YarnException {
+ // timelineServiceAddress could haven't be initialized yet
+ // or stale (only for new timeline service)
+ int retries = pollTimelineServiceAddress(this.maxServiceRetries);
+ if (timelineServiceAddress == null) {
+ String errMessage = "TimelineClient has reached to max retry times : "
+ + this.maxServiceRetries
+ + ", but failed to fetch timeline service address. Please verify"
+ + " Timeline Auxillary Service is configured in all the NMs";
+ LOG.error(errMessage);
+ throw new YarnException(errMessage);
+ }
+ return retries;
+ }
+
/**
* Check if reaching to maximum of retries.
* @param retries
@@ -641,7 +646,7 @@ public class TimelineClientImpl extends TimelineClient {
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
- timelineServiceAddress = getTimelineServiceAddress();
+ // timelineServiceAddress = getTimelineServiceAddress();
retries--;
}
return retries;
@@ -906,4 +911,212 @@ public class TimelineClientImpl extends TimelineClient {
}
}
+ private final class EntitiesHolder extends FutureTask<Void> {
+ private final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities entities;
+ private final boolean isSync;
+
+ EntitiesHolder(
+ final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities entities,
+ final boolean isSync) {
+ super(new Callable<Void>() {
+ // publishEntities()
+ public Void call() throws Exception {
+ MultivaluedMap<String, String> params = new MultivaluedMapImpl();
+ params.add("appid", contextAppId.toString());
+ params.add("async", Boolean.toString(!isSync));
+ putObjects("entities", params, entities);
+ return null;
+ }
+ });
+ this.entities = entities;
+ this.isSync = isSync;
+ }
+
+ public boolean isSync() {
+ return isSync;
+ }
+
+ public org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities getEntities() {
+ return entities;
+ }
+ }
+
+ /**
+ * This class is responsible for collecting the timeline entities and
+ * publishing them in async.
+ */
+ private class TimelineEntityDispatcher {
+ /**
+ * Time period for which the timelineclient will wait for draining after
+ * stop
+ */
+ private static final long DRAIN_TIME_PERIOD = 2000L;
+
+ private int numberOfAsyncsToMerge;
+ private final BlockingQueue<EntitiesHolder> timelineEntityQueue;
+ private ExecutorService executor;
+
+ TimelineEntityDispatcher(Configuration conf) {
+ timelineEntityQueue = new LinkedBlockingQueue<EntitiesHolder>();
+ numberOfAsyncsToMerge =
+ conf.getInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE,
+ YarnConfiguration.DEFAULT_NUMBER_OF_ASYNC_ENTITIES_TO_MERGE);
+ }
+
+ Runnable createRunnable() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ try {
+ EntitiesHolder entitiesHolder;
+ while (!Thread.currentThread().isInterrupted()) {
+ // Merge all the async calls and make one push, but if its sync
+ // call push immediately
+ try {
+ entitiesHolder = timelineEntityQueue.take();
+ } catch (InterruptedException ie) {
+ LOG.info("Timeline dispatcher thread was interrupted ");
+ Thread.currentThread().interrupt();
+ return;
+ }
+ if (entitiesHolder != null) {
+ publishWithoutBlockingOnQueue(entitiesHolder);
+ }
+ }
+ } finally {
+ if (!timelineEntityQueue.isEmpty()) {
+ LOG.info("Yet to publish " + timelineEntityQueue.size()
+ + " timelineEntities, draining them now. ");
+ }
+ // Try to drain the remaining entities to be published @ the max for
+ // 2 seconds
+ long timeTillweDrain =
+ System.currentTimeMillis() + DRAIN_TIME_PERIOD;
+ while (!timelineEntityQueue.isEmpty()) {
+ publishWithoutBlockingOnQueue(timelineEntityQueue.poll());
+ if (System.currentTimeMillis() > timeTillweDrain) {
+ // time elapsed stop publishing further....
+ if (!timelineEntityQueue.isEmpty()) {
+ LOG.warn("Time to drain elapsed! Remaining "
+ + timelineEntityQueue.size() + "timelineEntities will not"
+ + " be published");
+ // if some entities were not drained then we need interrupt
+ // the threads which had put sync EntityHolders to the queue.
+ EntitiesHolder nextEntityInTheQueue = null;
+ while ((nextEntityInTheQueue =
+ timelineEntityQueue.poll()) != null) {
+ nextEntityInTheQueue.cancel(true);
+ }
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Publishes the given EntitiesHolder and return immediately if sync
+ * call, else tries to fetch the EntitiesHolder from the queue in non
+ * blocking fashion and collate the Entities if possible before
+ * publishing through REST.
+ *
+ * @param entitiesHolder
+ */
+ private void publishWithoutBlockingOnQueue(
+ EntitiesHolder entitiesHolder) {
+ if (entitiesHolder.isSync()) {
+ entitiesHolder.run();
+ return;
+ }
+ int count = 1;
+ while (true) {
+ // loop till we find a sync put Entities or there is nothing
+ // to take
+ EntitiesHolder nextEntityInTheQueue = timelineEntityQueue.poll();
+ if (nextEntityInTheQueue == null) {
+ // Nothing in the queue just publish and get back to the
+ // blocked wait state
+ entitiesHolder.run();
+ break;
+ } else if (nextEntityInTheQueue.isSync()) {
+ // flush all the prev async entities first
+ entitiesHolder.run();
+ // and then flush the sync entity
+ nextEntityInTheQueue.run();
+ break;
+ } else {
+ // append all async entities together and then flush
+ entitiesHolder.getEntities().addEntities(
+ nextEntityInTheQueue.getEntities().getEntities());
+ count++;
+ if (count == numberOfAsyncsToMerge) {
+ // Flush the entities if the number of the async
+ // putEntites merged reaches the desired limit. To avoid
+ // collecting multiple entities and delaying for a long
+ // time.
+ entitiesHolder.run();
+ break;
+ }
+ }
+ }
+ }
+ };
+ }
+
+ public void dispatchEntities(boolean sync,
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity[] entitiesTobePublished)
+ throws YarnException {
+ if (executor.isShutdown()) {
+ throw new YarnException("Timeline client is in the process of stopping,"
+ + " not accepting any more TimelineEntities");
+ }
+
+ // wrap all TimelineEntity into TimelineEntities object
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities entities =
+ new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities();
+ for (org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity : entitiesTobePublished) {
+ entities.addEntity(entity);
+ }
+
+ // created a holder and place it in queue
+ EntitiesHolder entitiesHolder = new EntitiesHolder(entities, sync);
+ try {
+ timelineEntityQueue.put(entitiesHolder);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new YarnException(
+ "Failed while adding entity to the queue for publishing", e);
+ }
+
+ if (sync) {
+ // In sync call we need to wait till its published and if any error then
+ // throw it back
+ try {
+ entitiesHolder.get();
+ } catch (ExecutionException e) {
+ throw new YarnException("Failed while publishing entity",
+ e.getCause());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new YarnException("Interrupted while publishing entity", e);
+ }
+ }
+ }
+
+ public void start() {
+ executor = Executors.newSingleThreadExecutor();
+ executor.execute(createRunnable());
+ }
+
+ public void stop() {
+ LOG.info("Stopping TimelineClient.");
+ executor.shutdownNow();
+ try {
+ executor.awaitTermination(DRAIN_TIME_PERIOD, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ e.printStackTrace();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 1867e98..ca913b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2189,6 +2189,13 @@
<value>1000</value>
</property>
+ <property>
+ <description>Time line V2 client tries to merge these many number of
+ async entities (if available) and then call the REST ATS V2 API to submit.
+ </description>
+ <name>yarn.timeline-service.timeline-client.number-of-async-entities-to-merge</name>
+ <value>10</value>
+ </property>
<!-- Shared Cache Configuration -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
new file mode 100644
index 0000000..7803f94
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
@@ -0,0 +1,304 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.client.api.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestTimelineClientV2Impl {
+ private static final Log LOG =
+ LogFactory.getLog(TestTimelineClientV2Impl.class);
+ private TestV2TimelineClient client;
+ private static long TIME_TO_SLEEP = 150;
+
+ @Before
+ public void setup() {
+ YarnConfiguration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f);
+ conf.setInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE, 3);
+ client = createTimelineClient(conf);
+ }
+
+ private TestV2TimelineClient createTimelineClient(YarnConfiguration conf) {
+ ApplicationId id = ApplicationId.newInstance(0, 0);
+ TestV2TimelineClient client = new TestV2TimelineClient(id);
+ client.init(conf);
+ client.start();
+ return client;
+ }
+
+ private class TestV2TimelineClient extends TimelineClientImpl {
+ private boolean sleepBeforeReturn;
+ private boolean throwException;
+
+ private List<TimelineEntities> publishedEntities;
+
+ public TimelineEntities getPublishedEntities(int putIndex) {
+ Assert.assertTrue("Not So many entities Published",
+ putIndex < publishedEntities.size());
+ return publishedEntities.get(putIndex);
+ }
+
+ public void setSleepBeforeReturn(boolean sleepBeforeReturn) {
+ this.sleepBeforeReturn = sleepBeforeReturn;
+ }
+
+ public void setThrowException(boolean throwException) {
+ this.throwException = throwException;
+ }
+
+ public int getNumOfTimelineEntitiesPublished() {
+ return publishedEntities.size();
+ }
+
+ public TestV2TimelineClient(ApplicationId id) {
+ super(id);
+ publishedEntities = new ArrayList<TimelineEntities>();
+ }
+
+ protected void putObjects(String path,
+ MultivaluedMap<String, String> params, Object obj)
+ throws IOException, YarnException {
+ if (throwException) {
+ throw new YarnException("ActualException");
+ }
+ publishedEntities.add((TimelineEntities) obj);
+ if (sleepBeforeReturn) {
+ try {
+ Thread.sleep(TIME_TO_SLEEP);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testPostEntities() throws Exception {
+ try {
+ client.putEntities(generateEntity("1"));
+ } catch (YarnException e) {
+ Assert.fail("Exception is not expected");
+ }
+ }
+
+ @Test
+ public void testASyncCallMerge() throws Exception {
+ client.setSleepBeforeReturn(true);
+ try {
+ client.putEntitiesAsync(generateEntity("1"));
+ Thread.sleep(TIME_TO_SLEEP / 2);
+ // by the time first put response comes push 2 entities in the queue
+ client.putEntitiesAsync(generateEntity("2"));
+ client.putEntitiesAsync(generateEntity("3"));
+ } catch (YarnException e) {
+ Assert.fail("Exception is not expected");
+ }
+ for (int i = 0; i < 4; i++) {
+ if (client.getNumOfTimelineEntitiesPublished() == 2) {
+ break;
+ }
+ Thread.sleep(TIME_TO_SLEEP);
+ }
+ Assert.assertEquals("two merged TimelineEntities needs to be published", 2,
+ client.getNumOfTimelineEntitiesPublished());
+ TimelineEntities secondPublishedEntities = client.getPublishedEntities(1);
+ Assert.assertEquals(
+ "Merged TimelineEntities Object needs to 2 TimelineEntity Object", 2,
+ secondPublishedEntities.getEntities().size());
+ Assert.assertEquals("Order of Async Events Needs to be FIFO", "2",
+ secondPublishedEntities.getEntities().get(0).getId());
+ Assert.assertEquals("Order of Async Events Needs to be FIFO", "3",
+ secondPublishedEntities.getEntities().get(1).getId());
+ }
+
+ @Test
+ public void testSyncCall() throws Exception {
+ try {
+ // sync entity should not be be merged with Async
+ client.putEntities(generateEntity("1"));
+ client.putEntitiesAsync(generateEntity("2"));
+ client.putEntitiesAsync(generateEntity("3"));
+ // except for the sync call above 2 should be merged
+ client.putEntities(generateEntity("4"));
+ } catch (YarnException e) {
+ Assert.fail("Exception is not expected");
+ }
+ for (int i = 0; i < 4; i++) {
+ if (client.getNumOfTimelineEntitiesPublished() == 3) {
+ break;
+ }
+ Thread.sleep(TIME_TO_SLEEP);
+ }
+ printReceivedEntities();
+ Assert.assertEquals("TimelineEntities not published as desired", 3,
+ client.getNumOfTimelineEntitiesPublished());
+ TimelineEntities firstPublishedEntities = client.getPublishedEntities(0);
+ Assert.assertEquals("sync entities should not be merged with async", 1,
+ firstPublishedEntities.getEntities().size());
+
+ // test before pushing the sync entities asyncs are merged and pushed
+ TimelineEntities secondPublishedEntities = client.getPublishedEntities(1);
+ Assert.assertEquals(
+ "async entities should be merged before publishing sync", 2,
+ secondPublishedEntities.getEntities().size());
+ Assert.assertEquals("Order of Async Events Needs to be FIFO", "2",
+ secondPublishedEntities.getEntities().get(0).getId());
+ Assert.assertEquals("Order of Async Events Needs to be FIFO", "3",
+ secondPublishedEntities.getEntities().get(1).getId());
+
+ // test the last entity published is sync put
+ TimelineEntities thirdPublishedEntities = client.getPublishedEntities(2);
+ Assert.assertEquals("sync entities had to be published at the last", 1,
+ thirdPublishedEntities.getEntities().size());
+ Assert.assertEquals("Expected last sync Event is not proper", "4",
+ thirdPublishedEntities.getEntities().get(0).getId());
+ }
+
+ @Test
+ public void testExceptionCalls() throws Exception {
+ client.setThrowException(true);
+ try {
+ client.putEntitiesAsync(generateEntity("1"));
+ } catch (YarnException e) {
+ Assert.fail("Async calls are not expected to throw exception");
+ }
+
+ try {
+ client.putEntities(generateEntity("2"));
+ Assert.fail("Sync calls are expected to throw exception");
+ } catch (YarnException e) {
+ Assert.assertEquals("Same exception needs to be thrown",
+ "ActualException", e.getCause().getMessage());
+ }
+ }
+
+ @Test
+ public void testConfigurableNumberOfMerges() throws Exception {
+ client.setSleepBeforeReturn(true);
+ try {
+ // At max 3 entities need to be merged
+ client.putEntitiesAsync(generateEntity("1"));
+ client.putEntitiesAsync(generateEntity("2"));
+ client.putEntitiesAsync(generateEntity("3"));
+ client.putEntitiesAsync(generateEntity("4"));
+ client.putEntities(generateEntity("5"));
+ client.putEntitiesAsync(generateEntity("6"));
+ client.putEntitiesAsync(generateEntity("7"));
+ client.putEntitiesAsync(generateEntity("8"));
+ client.putEntitiesAsync(generateEntity("9"));
+ client.putEntitiesAsync(generateEntity("10"));
+ } catch (YarnException e) {
+ Assert.fail("No exception expected");
+ }
+ // not having the same logic here as it doesn't depend on how many times
+ // events are published.
+ Thread.sleep(2 * TIME_TO_SLEEP);
+ printReceivedEntities();
+ for (TimelineEntities publishedEntities : client.publishedEntities) {
+ Assert.assertTrue(
+ "Number of entities should not be greater than 3 for each publish,"
+ + " but was " + publishedEntities.getEntities().size(),
+ publishedEntities.getEntities().size() <= 3);
+ }
+ }
+
+ @Test
+ public void testAfterStop() throws Exception {
+ client.setSleepBeforeReturn(true);
+ try {
+ // At max 3 entities need to be merged
+ client.putEntities(generateEntity("1"));
+ for (int i = 2; i < 20; i++) {
+ client.putEntitiesAsync(generateEntity("" + i));
+ }
+ client.stop();
+ try {
+ client.putEntitiesAsync(generateEntity("50"));
+ Assert.fail("Exception expected");
+ } catch (YarnException e) {
+ // expected
+ }
+ } catch (YarnException e) {
+ Assert.fail("No exception expected");
+ }
+ // not having the same logic here as it doesn't depend on how many times
+ // events are published.
+ for (int i = 0; i < 5; i++) {
+ TimelineEntities publishedEntities =
+ client.publishedEntities.get(client.publishedEntities.size() - 1);
+ TimelineEntity timelineEntity = publishedEntities.getEntities()
+ .get(publishedEntities.getEntities().size() - 1);
+ if (!timelineEntity.getId().equals("19")) {
+ Thread.sleep(2 * TIME_TO_SLEEP);
+ }
+ }
+ printReceivedEntities();
+ TimelineEntities publishedEntities =
+ client.publishedEntities.get(client.publishedEntities.size() - 1);
+ TimelineEntity timelineEntity = publishedEntities.getEntities()
+ .get(publishedEntities.getEntities().size() - 1);
+ Assert.assertEquals("", "19", timelineEntity.getId());
+ }
+
+ private void printReceivedEntities() {
+ for (int i = 0; i < client.getNumOfTimelineEntitiesPublished(); i++) {
+ TimelineEntities publishedEntities = client.getPublishedEntities(i);
+ StringBuilder entitiesPerPublish = new StringBuilder();
+ ;
+ for (TimelineEntity entity : publishedEntities.getEntities()) {
+ entitiesPerPublish.append(entity.getId());
+ entitiesPerPublish.append(",");
+ }
+ LOG.info("Entities Published @ index " + i + " : "
+ + entitiesPerPublish.toString());
+ }
+ }
+
+ private static TimelineEntity generateEntity(String id) {
+ TimelineEntity entity = new TimelineEntity();
+ entity.setId(id);
+ entity.setType("testEntity");
+ entity.setCreatedTime(System.currentTimeMillis());
+ return entity;
+ }
+
+ @After
+ public void tearDown() {
+ if (client != null) {
+ client.stop();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f59c7433/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 34dfdd8..bf07600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -893,7 +893,9 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
Map<ApplicationId, String> knownCollectorsMap =
response.getAppCollectorsMap();
if (knownCollectorsMap == null) {
- LOG.warn("the collectors map is null");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No collectors to update RM");
+ }
} else {
Set<Map.Entry<ApplicationId, String>> rmKnownCollectors =
knownCollectorsMap.entrySet();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-4447. Provide a mechanism to
represent complex filters and parse them at the REST layer (Varun Saxena via
sjlee)
Posted by gt...@apache.org.
YARN-4447. Provide a mechanism to represent complex filters and parse them at the REST layer (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/047cde55
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/047cde55
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/047cde55
Branch: refs/heads/YARN-2928
Commit: 047cde55aa957a1288b88fa503ef1204b56c649b
Parents: 28ec2be
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon May 2 14:06:19 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:38:09 2016 -0700
----------------------------------------------------------------------
.../reader/TimelineParseConstants.java | 34 +
.../reader/TimelineParseException.java | 36 +
.../timelineservice/reader/TimelineParser.java | 37 +
.../reader/TimelineParserForCompareExpr.java | 300 ++++++
.../reader/TimelineParserForDataToRetrieve.java | 95 ++
.../reader/TimelineParserForEqualityExpr.java | 343 +++++++
.../reader/TimelineParserForExistFilters.java | 51 +
.../reader/TimelineParserForKVFilters.java | 78 ++
.../reader/TimelineParserForNumericFilters.java | 72 ++
.../TimelineParserForRelationFilters.java | 71 ++
.../reader/TimelineReaderWebServices.java | 220 ++++-
.../reader/TimelineReaderWebServicesUtils.java | 196 ++--
.../reader/filter/TimelineCompareFilter.java | 73 +-
.../reader/filter/TimelineExistsFilter.java | 49 +-
.../reader/filter/TimelineFilterList.java | 36 +
.../reader/filter/TimelineKeyValueFilter.java | 13 +
.../reader/filter/TimelineKeyValuesFilter.java | 61 +-
.../reader/filter/TimelinePrefixFilter.java | 37 +
.../reader/TestTimelineReaderWebServices.java | 14 +-
...stTimelineReaderWebServicesHBaseStorage.java | 900 +++++++++++++++++-
.../TestTimelineReaderWebServicesUtils.java | 923 +++++++++++++++++++
21 files changed, 3442 insertions(+), 197 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
new file mode 100644
index 0000000..662a102
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseConstants.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+/**
+ * Set of constants used while parsing filter expressions.
+ */
+final class TimelineParseConstants {
+ private TimelineParseConstants() {
+ }
+ static final String COMMA_DELIMITER = ",";
+ static final String COLON_DELIMITER = ":";
+ static final char NOT_CHAR = '!';
+ static final char SPACE_CHAR = ' ';
+ static final char OPENING_BRACKET_CHAR = '(';
+ static final char CLOSING_BRACKET_CHAR = ')';
+ static final char COMMA_CHAR = ',';
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java
new file mode 100644
index 0000000..8d4a5dc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParseException.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+/**
+ * Exception thrown to indicate that a timeline filter expression cannot be
+ * parsed.
+ */
+class TimelineParseException extends Exception {
+
+ private static final long serialVersionUID = 1L;
+
+ public TimelineParseException() {
+ super();
+ }
+
+ public TimelineParseException(String message) {
+ super(message);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java
new file mode 100644
index 0000000..6b461a0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParser.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.Closeable;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+
+@Private
+@Unstable
+interface TimelineParser extends Closeable {
+ /**
+ * Method used for parsing.
+ *
+ * @return a {@link TimelineFilterList} object.
+ * @throws TimelineParseException if any problem occurs while parsing.
+ */
+ TimelineFilterList parse() throws TimelineParseException;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java
new file mode 100644
index 0000000..1b020d9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForCompareExpr.java
@@ -0,0 +1,300 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.Deque;
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+
+/**
+ * Abstract class for parsing compare expressions.
+ * Compare expressions are of the form :
+ * (<key> <compareop> <value>) <op> (<key
+ * > <compareop> <value>)
+ * compareop is used to compare value of a the specified key in the backend
+ * storage. compareop can be :
+ * 1. eq - Equals
+ * 2. ne - Not equals (matches if key does not exist)
+ * 3. ene - Exists and not equals (key must exist for match to occur)
+ * 4. lt - Less than
+ * 5. gt - Greater than
+ * 6. le - Less than or equals
+ * 7. ge - Greater than or equals
+ * compareop's supported would depend on implementation. For instance, all
+ * the above compareops' will be supported for metric filters but only eq,ne and
+ * ene would be supported for KV filters like config/info filters.
+ *
+ * op is a logical operator and can be either AND or OR.
+ *
+ * The way values will be interpreted would also depend on implementation
+ *
+ * A typical compare expression would look as under:
+ * ((key1 eq val1 OR key2 ne val2) AND (key5 gt val45))
+ */
+@Private
+@Unstable
+abstract class TimelineParserForCompareExpr implements TimelineParser {
+ private enum ParseState {
+ PARSING_KEY,
+ PARSING_VALUE,
+ PARSING_OP,
+ PARSING_COMPAREOP
+ }
+ // Main expression.
+ private final String expr;
+ // Expression in lower case.
+ private final String exprInLowerCase;
+ private final String exprName;
+ private int offset = 0;
+ private int kvStartOffset = 0;
+ private final int exprLength;
+ private ParseState currentParseState = ParseState.PARSING_KEY;
+ // Linked list implemented as a stack.
+ private Deque<TimelineFilterList> filterListStack = new LinkedList<>();
+ private TimelineFilter currentFilter = null;
+ private TimelineFilterList filterList = null;
+ public TimelineParserForCompareExpr(String expression, String name) {
+ if (expression != null) {
+ expr = expression.trim();
+ exprLength = expr.length();
+ exprInLowerCase = expr.toLowerCase();
+ } else {
+ expr = null;
+ exprInLowerCase = null;
+ exprLength = 0;
+ }
+ this.exprName = name;
+ }
+
+ protected TimelineFilter getCurrentFilter() {
+ return currentFilter;
+ }
+
+ protected TimelineFilter getFilterList() {
+ return filterList;
+ }
+
+ protected abstract TimelineFilter createFilter();
+
+ protected abstract Object parseValue(String strValue)
+ throws TimelineParseException;
+
+ protected abstract void setCompareOpToCurrentFilter(
+ TimelineCompareOp compareOp, boolean keyMustExistFlag)
+ throws TimelineParseException;
+
+ protected abstract void setValueToCurrentFilter(Object value);
+
+ private void handleSpaceChar() throws TimelineParseException {
+ if (currentParseState == ParseState.PARSING_KEY ||
+ currentParseState == ParseState.PARSING_VALUE) {
+ if (kvStartOffset == offset) {
+ kvStartOffset++;
+ offset++;
+ return;
+ }
+ String str = expr.substring(kvStartOffset, offset);
+ if (currentParseState == ParseState.PARSING_KEY) {
+ if (currentFilter == null) {
+ currentFilter = createFilter();
+ }
+ ((TimelineCompareFilter)currentFilter).setKey(str);
+ currentParseState = ParseState.PARSING_COMPAREOP;
+ } else if (currentParseState == ParseState.PARSING_VALUE) {
+ if (currentFilter != null) {
+ setValueToCurrentFilter(parseValue(str));
+ }
+ currentParseState = ParseState.PARSING_OP;
+ }
+ }
+ offset++;
+ }
+
+ private void handleOpeningBracketChar() throws TimelineParseException {
+ if (currentParseState != ParseState.PARSING_KEY) {
+ throw new TimelineParseException("Encountered unexpected opening " +
+ "bracket while parsing " + exprName + ".");
+ }
+ offset++;
+ kvStartOffset = offset;
+ filterListStack.push(filterList);
+ filterList = null;
+ }
+
+ private void handleClosingBracketChar() throws TimelineParseException {
+ if (currentParseState != ParseState.PARSING_VALUE &&
+ currentParseState != ParseState.PARSING_OP) {
+ throw new TimelineParseException("Encountered unexpected closing " +
+ "bracket while parsing " + exprName + ".");
+ }
+ if (!filterListStack.isEmpty()) {
+ if (currentParseState == ParseState.PARSING_VALUE) {
+ setValueToCurrentFilter(
+ parseValue(expr.substring(kvStartOffset, offset)));
+ currentParseState = ParseState.PARSING_OP;
+ }
+ if (currentFilter != null) {
+ filterList.addFilter(currentFilter);
+ }
+ // As bracket is closing, pop the filter list from top of the stack and
+ // combine it with current filter list.
+ TimelineFilterList fList = filterListStack.pop();
+ if (fList != null) {
+ fList.addFilter(filterList);
+ filterList = fList;
+ }
+ currentFilter = null;
+ offset++;
+ kvStartOffset = offset;
+ } else {
+ throw new TimelineParseException("Encountered unexpected closing " +
+ "bracket while parsing " + exprName + ".");
+ }
+ }
+
+ private void parseCompareOp() throws TimelineParseException {
+ if (offset + 2 >= exprLength) {
+ throw new TimelineParseException("Compare op cannot be parsed for " +
+ exprName + ".");
+ }
+ TimelineCompareOp compareOp = null;
+ boolean keyExistFlag = true;
+ if (expr.charAt(offset + 2) == TimelineParseConstants.SPACE_CHAR) {
+ if (exprInLowerCase.startsWith("eq", offset)) {
+ compareOp = TimelineCompareOp.EQUAL;
+ } else if (exprInLowerCase.startsWith("ne", offset)) {
+ compareOp = TimelineCompareOp.NOT_EQUAL;
+ keyExistFlag = false;
+ } else if (exprInLowerCase.startsWith("lt", offset)) {
+ compareOp = TimelineCompareOp.LESS_THAN;
+ } else if (exprInLowerCase.startsWith("le", offset)) {
+ compareOp = TimelineCompareOp.LESS_OR_EQUAL;
+ } else if (exprInLowerCase.startsWith("gt", offset)) {
+ compareOp = TimelineCompareOp.GREATER_THAN;
+ } else if (exprInLowerCase.startsWith("ge", offset)) {
+ compareOp = TimelineCompareOp.GREATER_OR_EQUAL;
+ }
+ offset = offset + 3;
+ } else if (exprInLowerCase.startsWith("ene ", offset)) {
+ // Not equal but key should be present.
+ compareOp = TimelineCompareOp.NOT_EQUAL;
+ offset = offset + 4;
+ }
+ if (compareOp == null) {
+ throw new TimelineParseException("Compare op cannot be parsed for " +
+ exprName + ".");
+ }
+ setCompareOpToCurrentFilter(compareOp, keyExistFlag);
+ kvStartOffset = offset;
+ currentParseState = ParseState.PARSING_VALUE;
+ }
+
+ private void parseOp(boolean closingBracket) throws TimelineParseException {
+ Operator operator = null;
+ if (exprInLowerCase.startsWith("or ", offset)) {
+ operator = Operator.OR;
+ offset = offset + 3;
+ } else if (exprInLowerCase.startsWith("and ", offset)) {
+ operator = Operator.AND;
+ offset = offset + 4;
+ }
+ if (operator == null) {
+ throw new TimelineParseException("Operator cannot be parsed for " +
+ exprName + ".");
+ }
+ if (filterList == null) {
+ filterList = new TimelineFilterList(operator);
+ }
+ if (currentFilter != null) {
+ filterList.addFilter(currentFilter);
+ }
+ if (closingBracket || filterList.getOperator() != operator) {
+ filterList = new TimelineFilterList(operator, filterList);
+ }
+ currentFilter = null;
+ kvStartOffset = offset;
+ currentParseState = ParseState.PARSING_KEY;
+ }
+
+ @Override
+ public TimelineFilterList parse() throws TimelineParseException {
+ if (expr == null || exprLength == 0) {
+ return null;
+ }
+ boolean closingBracket = false;
+ while (offset < exprLength) {
+ char offsetChar = expr.charAt(offset);
+ switch(offsetChar) {
+ case TimelineParseConstants.SPACE_CHAR:
+ handleSpaceChar();
+ break;
+ case TimelineParseConstants.OPENING_BRACKET_CHAR:
+ handleOpeningBracketChar();
+ break;
+ case TimelineParseConstants.CLOSING_BRACKET_CHAR:
+ handleClosingBracketChar();
+ closingBracket = true;
+ break;
+ default: // other characters.
+ // Parse based on state.
+ if (currentParseState == ParseState.PARSING_COMPAREOP) {
+ parseCompareOp();
+ } else if (currentParseState == ParseState.PARSING_OP) {
+ parseOp(closingBracket);
+ closingBracket = false;
+ } else {
+ // Might be a key or value. Move ahead.
+ offset++;
+ }
+ break;
+ }
+ }
+ if (!filterListStack.isEmpty()) {
+ filterListStack.clear();
+ throw new TimelineParseException("Encountered improper brackets while " +
+ "parsing " + exprName + ".");
+ }
+ if (currentParseState == ParseState.PARSING_VALUE) {
+ setValueToCurrentFilter(
+ parseValue(expr.substring(kvStartOffset, offset)));
+ }
+ if (filterList == null || filterList.getFilterList().isEmpty()) {
+ filterList = new TimelineFilterList(currentFilter);
+ } else if (currentFilter != null) {
+ filterList.addFilter(currentFilter);
+ }
+ return filterList;
+ }
+
+ @Override
+ public void close() {
+ if (filterListStack != null) {
+ filterListStack.clear();
+ }
+ filterList = null;
+ currentFilter = null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java
new file mode 100644
index 0000000..1e6039d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForDataToRetrieve.java
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+
+/**
+ * Used for parsing metrics or configs to retrieve.
+ */
+@Private
+@Unstable
+public class TimelineParserForDataToRetrieve implements TimelineParser {
+ private String expr;
+ private final int exprLength;
+ public TimelineParserForDataToRetrieve(String expression) {
+ this.expr = expression;
+ if (expression != null) {
+ this.expr = expr.trim();
+ exprLength = expr.length();
+ } else {
+ exprLength = 0;
+ }
+ }
+
+ @Override
+ public TimelineFilterList parse() throws TimelineParseException {
+ if (expr == null || exprLength == 0) {
+ return null;
+ }
+ TimelineCompareOp compareOp = null;
+ int openingBracketIndex =
+ expr.indexOf(TimelineParseConstants.OPENING_BRACKET_CHAR);
+ if (expr.charAt(0) == TimelineParseConstants.NOT_CHAR) {
+ if (openingBracketIndex == -1) {
+ throw new TimelineParseException("Invalid config/metric to retrieve " +
+ "expression");
+ }
+ if (openingBracketIndex != 1 &&
+ expr.substring(1, openingBracketIndex + 1).trim().length() != 1) {
+ throw new TimelineParseException("Invalid config/metric to retrieve " +
+ "expression");
+ }
+ compareOp = TimelineCompareOp.NOT_EQUAL;
+ } else if (openingBracketIndex <= 0) {
+ compareOp = TimelineCompareOp.EQUAL;
+ }
+ char lastChar = expr.charAt(exprLength - 1);
+ if (compareOp == TimelineCompareOp.NOT_EQUAL &&
+ lastChar != TimelineParseConstants.CLOSING_BRACKET_CHAR) {
+ throw new TimelineParseException("Invalid config/metric to retrieve " +
+ "expression");
+ }
+ if (openingBracketIndex != -1 &&
+ expr.charAt(exprLength - 1) ==
+ TimelineParseConstants.CLOSING_BRACKET_CHAR) {
+ expr = expr.substring(openingBracketIndex + 1, exprLength - 1).trim();
+ }
+ if (expr.isEmpty()) {
+ return null;
+ }
+ Operator op =
+ (compareOp == TimelineCompareOp.NOT_EQUAL) ? Operator.AND : Operator.OR;
+ TimelineFilterList list = new TimelineFilterList(op);
+ String[] splits = expr.split(TimelineParseConstants.COMMA_DELIMITER);
+ for (String split : splits) {
+ list.addFilter(new TimelinePrefixFilter(compareOp, split.trim()));
+ }
+ return list;
+ }
+
+ @Override
+ public void close() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java
new file mode 100644
index 0000000..7451713
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForEqualityExpr.java
@@ -0,0 +1,343 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.Deque;
+import java.util.LinkedList;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+
+/**
+ * Abstract class for parsing equality expressions. This means the values in
+ * expression would either be equal or not equal.
+ * Equality expressions are of the form :
+ * (<value>,<value>,<value>) <op> !(<value>,
+ * <value>)
+ *
+ * Here, "!" means all the values should not exist/should not be equal.
+ * If not specified, they should exist/be equal.
+ *
+ * op is a logical operator and can be either AND or OR.
+ *
+ * The way values will be interpreted would also depend on implementation.
+ *
+ * For instance for event filters this expression may look like,
+ * (event1,event2) AND !(event3,event4)
+ * This means for an entity to match, event1 and event2 should exist. But event3
+ * and event4 should not exist.
+ */
+@Private
+@Unstable
+abstract class TimelineParserForEqualityExpr implements TimelineParser {
+ private enum ParseState {
+ PARSING_VALUE,
+ PARSING_OP,
+ PARSING_COMPAREOP
+ }
+ private final String expr;
+ // Expression in lower case.
+ private final String exprInLowerCase;
+ // Expression name.
+ private final String exprName;
+ // Expression offset.
+ private int offset = 0;
+ // Offset used to parse values in the expression.
+ private int startOffset = 0;
+ private final int exprLength;
+ private ParseState currentParseState = ParseState.PARSING_COMPAREOP;
+ private TimelineCompareOp currentCompareOp = null;
+ // Used to store filter lists which can then be combined as brackets are
+ // closed.
+ private Deque<TimelineFilterList> filterListStack = new LinkedList<>();
+ private TimelineFilter currentFilter = null;
+ private TimelineFilterList filterList = null;
+ // Delimiter used to separate values.
+ private final char delimiter;
+ public TimelineParserForEqualityExpr(String expression, String name,
+ char delim) {
+ if (expression != null) {
+ expr = expression.trim();
+ exprLength = expr.length();
+ exprInLowerCase = expr.toLowerCase();
+ } else {
+ exprLength = 0;
+ expr = null;
+ exprInLowerCase = null;
+ }
+ exprName = name;
+ delimiter = delim;
+ }
+
+ protected TimelineFilter getCurrentFilter() {
+ return currentFilter;
+ }
+
+ protected TimelineFilter getFilterList() {
+ return filterList;
+ }
+
+ /**
+ * Creates filter as per implementation.
+ *
+ * @return a {@link TimelineFilter} implementation.
+ */
+ protected abstract TimelineFilter createFilter();
+
+ /**
+ * Sets compare op to the current filter as per filter implementation.
+ *
+ * @param compareOp compare op to be set.
+ * @throws Exception if any problem occurs.
+ */
+ protected abstract void setCompareOpToCurrentFilter(
+ TimelineCompareOp compareOp) throws TimelineParseException;
+
+ /**
+ * Sets value to the current filter as per filter implementation.
+ *
+ * @param value value to be set.
+ * @throws Exception if any problem occurs.
+ */
+ protected abstract void setValueToCurrentFilter(String value)
+ throws TimelineParseException;
+
+ private void createAndSetFilter(boolean checkIfNull)
+ throws TimelineParseException {
+ if (!checkIfNull || currentFilter == null) {
+ currentFilter = createFilter();
+ setCompareOpToCurrentFilter(currentCompareOp);
+ }
+ setValueToCurrentFilter(expr.substring(startOffset, offset).trim());
+ }
+
+ private void handleSpaceChar() throws TimelineParseException {
+ if (currentParseState == ParseState.PARSING_VALUE) {
+ if (startOffset == offset) {
+ startOffset++;
+ } else {
+ createAndSetFilter(true);
+ currentParseState = ParseState.PARSING_OP;
+ }
+ }
+ offset++;
+ }
+
+ private void handleDelimiter() throws TimelineParseException {
+ if (currentParseState == ParseState.PARSING_OP ||
+ currentParseState == ParseState.PARSING_VALUE) {
+ if (currentParseState == ParseState.PARSING_VALUE) {
+ createAndSetFilter(false);
+ }
+ if (filterList == null) {
+ filterList = new TimelineFilterList();
+ }
+ // Add parsed filter into filterlist and make it null to move on to next
+ // filter.
+ filterList.addFilter(currentFilter);
+ currentFilter = null;
+ offset++;
+ startOffset = offset;
+ currentParseState = ParseState.PARSING_VALUE;
+ } else {
+ throw new TimelineParseException("Invalid " + exprName + "expression.");
+ }
+ }
+
+ private void handleOpeningBracketChar(boolean encounteredNot)
+ throws TimelineParseException {
+ if (currentParseState == ParseState.PARSING_COMPAREOP ||
+ currentParseState == ParseState.PARSING_VALUE) {
+ offset++;
+ startOffset = offset;
+ filterListStack.push(filterList);
+ filterList = null;
+ if (currentFilter == null) {
+ currentFilter = createFilter();
+ }
+ currentCompareOp = encounteredNot ?
+ TimelineCompareOp.NOT_EQUAL : TimelineCompareOp.EQUAL;
+ setCompareOpToCurrentFilter(currentCompareOp);
+ currentParseState = ParseState.PARSING_VALUE;
+ } else {
+ throw new TimelineParseException("Encountered unexpected opening " +
+ "bracket while parsing " + exprName + ".");
+ }
+ }
+
+ private void handleNotChar() throws TimelineParseException {
+ if (currentParseState == ParseState.PARSING_COMPAREOP ||
+ currentParseState == ParseState.PARSING_VALUE) {
+ offset++;
+ while (offset < exprLength &&
+ expr.charAt(offset) == TimelineParseConstants.SPACE_CHAR) {
+ offset++;
+ }
+ if (offset == exprLength) {
+ throw new TimelineParseException("Invalid " + exprName + "expression");
+ }
+ if (expr.charAt(offset) == TimelineParseConstants.OPENING_BRACKET_CHAR) {
+ handleOpeningBracketChar(true);
+ } else {
+ throw new TimelineParseException("Invalid " + exprName + "expression");
+ }
+ } else {
+ throw new TimelineParseException("Encountered unexpected not(!) char " +
+ "while parsing " + exprName + ".");
+ }
+ }
+
+ private void handleClosingBracketChar() throws TimelineParseException {
+ if (currentParseState != ParseState.PARSING_VALUE &&
+ currentParseState != ParseState.PARSING_OP) {
+ throw new TimelineParseException("Encountered unexpected closing " +
+ "bracket while parsing " + exprName + ".");
+ }
+ if (!filterListStack.isEmpty()) {
+ if (currentParseState == ParseState.PARSING_VALUE) {
+ if (startOffset != offset) {
+ createAndSetFilter(true);
+ currentParseState = ParseState.PARSING_OP;
+ }
+ }
+ if (filterList == null) {
+ filterList = new TimelineFilterList();
+ }
+ if (currentFilter != null) {
+ filterList.addFilter(currentFilter);
+ }
+ // As bracket is closing, pop the filter list from top of the stack and
+ // combine it with current filter list.
+ TimelineFilterList fList = filterListStack.pop();
+ if (fList != null) {
+ fList.addFilter(filterList);
+ filterList = fList;
+ }
+ currentFilter = null;
+ offset++;
+ startOffset = offset;
+ } else {
+ throw new TimelineParseException("Encountered unexpected closing " +
+ "bracket while parsing " + exprName + ".");
+ }
+ }
+
+ private void parseOp(boolean closingBracket) throws TimelineParseException {
+ Operator operator = null;
+ if (exprInLowerCase.startsWith("or ", offset)) {
+ operator = Operator.OR;
+ offset = offset + 3;
+ } else if (exprInLowerCase.startsWith("and ", offset)) {
+ operator = Operator.AND;
+ offset = offset + 4;
+ }
+ if (operator == null) {
+ throw new TimelineParseException("Operator cannot be parsed for " +
+ exprName + ".");
+ }
+ if (filterList == null) {
+ filterList = new TimelineFilterList(operator);
+ }
+ if (currentFilter != null) {
+ filterList.addFilter(currentFilter);
+ }
+ if (closingBracket || filterList.getOperator() != operator) {
+ filterList = new TimelineFilterList(operator, filterList);
+ }
+ currentFilter = null;
+ startOffset = offset;
+ currentParseState = ParseState.PARSING_COMPAREOP;
+ }
+
+ private void parseCompareOp() throws TimelineParseException {
+ if (currentFilter == null) {
+ currentFilter = createFilter();
+ }
+ currentCompareOp = TimelineCompareOp.EQUAL;
+ setCompareOpToCurrentFilter(currentCompareOp);
+ currentParseState = ParseState.PARSING_VALUE;
+ }
+
+ @Override
+ public TimelineFilterList parse() throws TimelineParseException {
+ if (expr == null || exprLength == 0) {
+ return null;
+ }
+ boolean closingBracket = false;
+ while (offset < exprLength) {
+ char offsetChar = expr.charAt(offset);
+ switch(offsetChar) {
+ case TimelineParseConstants.NOT_CHAR:
+ handleNotChar();
+ break;
+ case TimelineParseConstants.SPACE_CHAR:
+ handleSpaceChar();
+ break;
+ case TimelineParseConstants.OPENING_BRACKET_CHAR:
+ handleOpeningBracketChar(false);
+ break;
+ case TimelineParseConstants.CLOSING_BRACKET_CHAR:
+ handleClosingBracketChar();
+ closingBracket = true;
+ break;
+ default: // other characters.
+ if (offsetChar == delimiter) {
+ handleDelimiter();
+ } else if (currentParseState == ParseState.PARSING_COMPAREOP) {
+ parseCompareOp();
+ } else if (currentParseState == ParseState.PARSING_OP) {
+ parseOp(closingBracket);
+ closingBracket = false;
+ } else {
+ offset++;
+ }
+ break;
+ }
+ }
+ if (!filterListStack.isEmpty()) {
+ filterListStack.clear();
+ throw new TimelineParseException("Encountered improper brackets while " +
+ "parsing " + exprName + ".");
+ }
+ if (currentParseState == ParseState.PARSING_VALUE) {
+ if (startOffset != offset) {
+ createAndSetFilter(true);
+ }
+ }
+ if (filterList == null || filterList.getFilterList().isEmpty()) {
+ filterList = new TimelineFilterList(currentFilter);
+ } else if (currentFilter != null) {
+ filterList.addFilter(currentFilter);
+ }
+ return filterList;
+ }
+
+ @Override
+ public void close() {
+ if (filterListStack != null) {
+ filterListStack.clear();
+ }
+ currentFilter = null;
+ filterList = null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java
new file mode 100644
index 0000000..8048c6e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForExistFilters.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+
+/**
+ * Used for parsing existence filters such as event filters. These filters
+ * check for existence of a value. For example, in case of event filters, they
+ * check if an event exists or not and accordingly return an entity.
+ */
+@Private
+@Unstable
+class TimelineParserForExistFilters extends TimelineParserForEqualityExpr {
+
+ public TimelineParserForExistFilters(String expression, char delimiter) {
+ super(expression, "Event Filter", delimiter);
+ }
+
+ protected TimelineFilter createFilter() {
+ return new TimelineExistsFilter();
+ }
+
+ protected void setValueToCurrentFilter(String value) {
+ ((TimelineExistsFilter)getCurrentFilter()).setValue(value);
+ }
+
+ protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp) {
+ ((TimelineExistsFilter)getCurrentFilter()).setCompareOp(compareOp);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java
new file mode 100644
index 0000000..ec68bec
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForKVFilters.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+
+/**
+ * Used for parsing key-value filters such as config and info filters.
+ */
+@Private
+@Unstable
+class TimelineParserForKVFilters extends TimelineParserForCompareExpr {
+ // Indicates if value has to be interpreted as a string.
+ private final boolean valueAsString;
+ public TimelineParserForKVFilters(String expression, boolean valAsStr) {
+ super(expression, "Config/Info Filter");
+ this.valueAsString = valAsStr;
+ }
+
+ protected TimelineFilter createFilter() {
+ return new TimelineKeyValueFilter();
+ }
+
+ protected Object parseValue(String strValue) {
+ if (!valueAsString) {
+ try {
+ return GenericObjectMapper.OBJECT_READER.readValue(strValue);
+ } catch (IOException e) {
+ return strValue;
+ }
+ } else {
+ return strValue;
+ }
+ }
+
+ @Override
+ protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp,
+ boolean keyMustExistFlag) throws TimelineParseException {
+ if (compareOp != TimelineCompareOp.EQUAL &&
+ compareOp != TimelineCompareOp.NOT_EQUAL) {
+ throw new TimelineParseException("TimelineCompareOp for kv-filter " +
+ "should be EQUAL or NOT_EQUAL");
+ }
+ ((TimelineKeyValueFilter)getCurrentFilter()).setCompareOp(
+ compareOp, keyMustExistFlag);
+ }
+
+ @Override
+ protected void setValueToCurrentFilter(Object value) {
+ TimelineFilter currentFilter = getCurrentFilter();
+ if (currentFilter != null) {
+ ((TimelineKeyValueFilter)currentFilter).setValue(value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java
new file mode 100644
index 0000000..7c14a9f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForNumericFilters.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+
+/**
+ * Used for parsing numerical filters such as metric filters.
+ */
+@Private
+@Unstable
+class TimelineParserForNumericFilters extends TimelineParserForCompareExpr {
+
+ public TimelineParserForNumericFilters(String expression) {
+ super(expression, "Metric Filter");
+ }
+
+ protected TimelineFilter createFilter() {
+ return new TimelineCompareFilter();
+ }
+
+ @Override
+ protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp,
+ boolean keyMustExistFlag) {
+ ((TimelineCompareFilter)getCurrentFilter()).setCompareOp(
+ compareOp, keyMustExistFlag);
+ }
+
+ protected Object parseValue(String strValue) throws TimelineParseException {
+ Object value = null;
+ try {
+ value = GenericObjectMapper.OBJECT_READER.readValue(strValue);
+ } catch (IOException e) {
+ throw new TimelineParseException("Value cannot be parsed.");
+ }
+ if (value == null || !(TimelineStorageUtils.isIntegralValue(value))) {
+ throw new TimelineParseException("Value is not a number.");
+ }
+ return value;
+ }
+
+ protected void setValueToCurrentFilter(Object value) {
+ TimelineFilter currentFilter = getCurrentFilter();
+ if (currentFilter != null) {
+ ((TimelineCompareFilter)currentFilter).setValue(value);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java
new file mode 100644
index 0000000..cde11e4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineParserForRelationFilters.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
+
+/**
+ * Used for parsing relation filters.
+ */
+@Private
+@Unstable
+class TimelineParserForRelationFilters extends
+ TimelineParserForEqualityExpr {
+ private final String valueDelimiter;
+ public TimelineParserForRelationFilters(String expression, char valuesDelim,
+ String valueDelim) {
+ super(expression, "Relation Filter", valuesDelim);
+ valueDelimiter = valueDelim;
+ }
+
+ @Override
+ protected TimelineFilter createFilter() {
+ return new TimelineKeyValuesFilter();
+ }
+
+ @Override
+ protected void setCompareOpToCurrentFilter(TimelineCompareOp compareOp) {
+ ((TimelineKeyValuesFilter)getCurrentFilter()).setCompareOp(compareOp);
+ }
+
+ @Override
+ protected void setValueToCurrentFilter(String value)
+ throws TimelineParseException {
+ if (value != null) {
+ String[] pairStrs = value.split(valueDelimiter);
+ if (pairStrs.length < 2) {
+ throw new TimelineParseException("Invalid relation filter expression");
+ }
+ String key = pairStrs[0].trim();
+ Set<Object> values = new HashSet<Object>();
+ for (int i = 1; i < pairStrs.length; i++) {
+ values.add(pairStrs[i].trim());
+ }
+ ((TimelineKeyValuesFilter)getCurrentFilter()).
+ setKeyAndValues(key, values);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 0b9549b..6ec59ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -178,6 +178,9 @@ public class TimelineReaderWebServices {
"Requested Invalid Field." : e.getMessage());
} else if (e instanceof NotFoundException) {
throw (NotFoundException)e;
+ } else if (e instanceof TimelineParseException) {
+ throw new BadRequestException(e.getMessage() == null ?
+ "Filter Parsing failed." : e.getMessage());
} else if (e instanceof BadRequestException) {
throw (BadRequestException)e;
} else {
@@ -239,6 +242,14 @@ public class TimelineReaderWebServices {
* metricfilters=metricid1, metricid2... (Optional query param).
* @param eventfilters If specified, matched entities should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type, id and created time is returned
@@ -270,6 +281,8 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -295,7 +308,7 @@ public class TimelineReaderWebServices {
limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
infofilters, conffilters, metricfilters, eventfilters),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -354,6 +367,14 @@ public class TimelineReaderWebServices {
* metricfilters=metricid1, metricid2... (Optional query param).
* @param eventfilters If specified, matched entities should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type, id, created time is returned
@@ -390,11 +411,13 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, appId, entityType, userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
- fields);
+ confsToRetrieve, metricsToRetrieve, fields);
}
/**
@@ -443,6 +466,14 @@ public class TimelineReaderWebServices {
* metricfilters=metricid1, metricid2... (Optional query param).
* @param eventfilters If specified, matched entities should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type, id, created time is returned
@@ -480,6 +511,8 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -496,12 +529,11 @@ public class TimelineReaderWebServices {
entities = timelineReaderManager.getEntities(
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, flowRunId, appId, entityType, null),
-
TimelineReaderWebServicesUtils.createTimelineEntityFilters(
limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
infofilters, conffilters, metricfilters, eventfilters),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -524,6 +556,14 @@ public class TimelineReaderWebServices {
* @param uId a delimited string containing clusterid, userid, flow name,
* flowrun id, app id, entity type and entity id which are extracted from
* UID and then used to query backend(Mandatory path param).
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type, id, created time is returned
@@ -546,6 +586,8 @@ public class TimelineReaderWebServices {
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("uid") String uId,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -566,7 +608,7 @@ public class TimelineReaderWebServices {
}
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -601,6 +643,14 @@ public class TimelineReaderWebServices {
* param).
* @param flowRunId Run id which should match for the entity(Optional query
* param).
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type, id, created time is returned
@@ -628,9 +678,11 @@ public class TimelineReaderWebServices {
@QueryParam("userid") String userId,
@QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getEntity(req, res, null, appId, entityType, entityId, userId,
- flowName, flowRunId, fields);
+ flowName, flowRunId, confsToRetrieve, metricsToRetrieve, fields);
}
/**
@@ -653,6 +705,14 @@ public class TimelineReaderWebServices {
* param).
* @param flowRunId Run id which should match for the entity(Optional query
* param).
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type, id and created time is returned
@@ -681,6 +741,8 @@ public class TimelineReaderWebServices {
@QueryParam("userid") String userId,
@QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -698,7 +760,7 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, flowRunId, appId, entityType, entityId),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -723,6 +785,8 @@ public class TimelineReaderWebServices {
* @param uId a delimited string containing clusterid, userid, flow name and
* flowrun id which are extracted from UID and then used to query backend
* (Mandatory path param).
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response.
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
* <cite>FlowRunEntity</cite> instance is returned. By default, all
@@ -741,7 +805,8 @@ public class TimelineReaderWebServices {
public TimelineEntity getFlowRun(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
- @PathParam("uid") String uId) {
+ @PathParam("uid") String uId,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
@@ -761,7 +826,8 @@ public class TimelineReaderWebServices {
}
context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
entity = timelineReaderManager.getEntity(context,
- new TimelineDataToRetrieve());
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, metricsToRetrieve, null));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -787,6 +853,8 @@ public class TimelineReaderWebServices {
* @param flowName Flow name to which the flow run to be queried belongs to(
* Mandatory path param).
* @param flowRunId Id of the flow run to be queried(Mandatory path param).
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response.
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
* <cite>FlowRunEntity</cite> instance is returned. By default, all
@@ -807,8 +875,10 @@ public class TimelineReaderWebServices {
@Context HttpServletResponse res,
@PathParam("userid") String userId,
@PathParam("flowname") String flowName,
- @PathParam("flowrunid") String flowRunId) {
- return getFlowRun(req, res, null, userId, flowName, flowRunId);
+ @PathParam("flowrunid") String flowRunId,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve) {
+ return getFlowRun(req, res, null, userId, flowName, flowRunId,
+ metricsToRetrieve);
}
/**
@@ -823,6 +893,8 @@ public class TimelineReaderWebServices {
* @param flowName Flow name to which the flow run to be queried belongs to(
* Mandatory path param).
* @param flowRunId Id of the flow run to be queried(Mandatory path param).
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response.
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
* <cite>FlowRunEntity</cite> instance is returned. By default, all
@@ -845,7 +917,8 @@ public class TimelineReaderWebServices {
@PathParam("clusterid") String clusterId,
@PathParam("userid") String userId,
@PathParam("flowname") String flowName,
- @PathParam("flowrunid") String flowRunId) {
+ @PathParam("flowrunid") String flowRunId,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
@@ -862,7 +935,8 @@ public class TimelineReaderWebServices {
TimelineReaderWebServicesUtils.createTimelineReaderContext(
clusterId, userId, flowName, flowRunId, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null),
- new TimelineDataToRetrieve());
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, metricsToRetrieve, null));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -894,6 +968,10 @@ public class TimelineReaderWebServices {
* created before this timestamp(Optional query param).
* @param createdTimeEnd If specified, matched flow runs should not be created
* after this timestamp(Optional query param).
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields to retrieve, see {@link Field}.
* All fields will be retrieved if fields=ALL. Fields other than METRICS
* have no meaning for this REST endpoint. If not specified, all fields
@@ -918,6 +996,7 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -942,7 +1021,7 @@ public class TimelineReaderWebServices {
limit, createdTimeStart, createdTimeEnd, null, null, null,
null, null, null),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ null, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
}
@@ -970,6 +1049,10 @@ public class TimelineReaderWebServices {
* created before this timestamp(Optional query param).
* @param createdTimeEnd If specified, matched flow runs should not be created
* after this timestamp(Optional query param).
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields to retrieve, see {@link Field}.
* All fields will be retrieved if fields=ALL. Fields other than METRICS
* have no meaning for this REST endpoint. If not specified, all fields
@@ -995,9 +1078,10 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getFlowRuns(req, res, null, userId, flowName, limit,
- createdTimeStart, createdTimeEnd, fields);
+ createdTimeStart, createdTimeEnd, metricsToRetrieve, fields);
}
/**
@@ -1016,6 +1100,10 @@ public class TimelineReaderWebServices {
* created before this timestamp(Optional query param).
* @param createdTimeEnd If specified, matched flow runs should not be created
* after this timestamp(Optional query param).
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields to retrieve, see {@link Field}.
* All fields will be retrieved if fields=ALL. Fields other than METRICS
* have no meaning for this REST endpoint. If not specified, all fields
@@ -1042,6 +1130,7 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -1063,7 +1152,7 @@ public class TimelineReaderWebServices {
limit, createdTimeStart, createdTimeEnd, null, null, null,
null, null, null),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ null, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
}
@@ -1204,6 +1293,14 @@ public class TimelineReaderWebServices {
* @param uId a delimited string containing clusterid, userid, flow name, flow
* run id and app id which are extracted from UID and then used to query
* backend(Mandatory path param).
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1226,6 +1323,8 @@ public class TimelineReaderWebServices {
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("uid") String uId,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -1247,7 +1346,7 @@ public class TimelineReaderWebServices {
context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString());
entity = timelineReaderManager.getEntity(context,
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1277,6 +1376,14 @@ public class TimelineReaderWebServices {
* @param flowRunId Run id which should match for the app(Optional query
* param).
* @param userId User id which should match for the app(Optional query param).
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1302,8 +1409,11 @@ public class TimelineReaderWebServices {
@QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("userid") String userId,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
- return getApp(req, res, null, appId, flowName, flowRunId, userId, fields);
+ return getApp(req, res, null, appId, flowName, flowRunId, userId,
+ confsToRetrieve, metricsToRetrieve, fields);
}
/**
@@ -1322,6 +1432,14 @@ public class TimelineReaderWebServices {
* @param flowRunId Run id which should match for the app(Optional query
* param).
* @param userId User id which should match for the app(Optional query param).
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1348,6 +1466,8 @@ public class TimelineReaderWebServices {
@QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("userid") String userId,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -1366,7 +1486,7 @@ public class TimelineReaderWebServices {
clusterId, userId, flowName, flowRunId, appId,
TimelineEntityType.YARN_APPLICATION.toString(), null),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1417,6 +1537,14 @@ public class TimelineReaderWebServices {
* (Optional query param).
* @param eventfilters If specified, matched apps should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1447,6 +1575,8 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
@@ -1471,7 +1601,7 @@ public class TimelineReaderWebServices {
limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
infofilters, conffilters, metricfilters, eventfilters),
TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
- null, null, fields));
+ confsToRetrieve, metricsToRetrieve, fields));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -1523,6 +1653,14 @@ public class TimelineReaderWebServices {
* (Optional query param).
* @param eventfilters If specified, matched apps should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1555,12 +1693,14 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
- fields);
+ confsToRetrieve, metricsToRetrieve, fields);
}
/**
@@ -1602,6 +1742,14 @@ public class TimelineReaderWebServices {
* (Optional query param).
* @param eventfilters If specified, matched apps should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1636,12 +1784,14 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
- fields);
+ confsToRetrieve, metricsToRetrieve, fields);
}
/**
@@ -1680,6 +1830,14 @@ public class TimelineReaderWebServices {
* (Optional query param).
* @param eventfilters If specified, matched apps should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1711,11 +1869,14 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
- infofilters, conffilters, metricfilters, eventfilters, fields);
+ infofilters, conffilters, metricfilters, eventfilters,
+ confsToRetrieve, metricsToRetrieve, fields);
}
/**
@@ -1756,6 +1917,14 @@ public class TimelineReaderWebServices {
* (Optional query param).
* @param eventfilters If specified, matched apps should contain the given
* events. This is represented as eventfilters=eventid1, eventid2...
+ * @param confsToRetrieve If specified, defines which configurations to
+ * retrieve and send back in response. These configs will be retrieved
+ * irrespective of whether configs are specified in fields to retrieve or
+ * not.
+ * @param metricsToRetrieve If specified, defines which metrics to retrieve
+ * and send back in response. These metrics will be retrieved
+ * irrespective of whether metrics are specified in fields to retrieve or
+ * not.
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
* specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
@@ -1788,10 +1957,13 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
+ @QueryParam("confstoretrieve") String confsToRetrieve,
+ @QueryParam("metricstoretrieve") String metricsToRetrieve,
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
- infofilters, conffilters, metricfilters, eventfilters, fields);
+ infofilters, conffilters, metricfilters, eventfilters,
+ confsToRetrieve, metricsToRetrieve, fields);
}
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-4445. Unify the term flowId
and flowName in timeline v2 codebase. Contributed by Zhan Zhang.
Posted by gt...@apache.org.
YARN-4445. Unify the term flowId and flowName in timeline v2 codebase.
Contributed by Zhan Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89b4101f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89b4101f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89b4101f
Branch: refs/heads/YARN-2928
Commit: 89b4101f2c8b1e9473a8352b0412c6ea83066caf
Parents: 4764f4a
Author: Li Lu <gt...@apache.org>
Authored: Wed Dec 16 16:42:57 2015 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:17:12 2016 -0700
----------------------------------------------------------------------
.../mapred/TestMRTimelineEventHandling.java | 2 +-
.../distributedshell/TestDistributedShell.java | 2 +-
.../yarn/util/timeline/TimelineUtils.java | 2 +-
.../TestSystemMetricsPublisherForV2.java | 2 +-
.../collector/AppLevelTimelineCollector.java | 2 +-
.../reader/TimelineReaderManager.java | 8 +--
.../reader/TimelineReaderWebServices.java | 74 ++++++++++----------
.../storage/ApplicationEntityReader.java | 20 +++---
.../storage/FileSystemTimelineReaderImpl.java | 14 ++--
.../storage/FlowActivityEntityReader.java | 10 +--
.../storage/FlowRunEntityReader.java | 16 ++---
.../storage/GenericEntityReader.java | 22 +++---
.../storage/HBaseTimelineReaderImpl.java | 8 +--
.../storage/TimelineEntityReader.java | 10 +--
.../storage/TimelineEntityReaderFactory.java | 20 +++---
.../timelineservice/storage/TimelineReader.java | 8 +--
.../storage/application/ApplicationRowKey.java | 38 +++++-----
.../storage/application/ApplicationTable.java | 2 +-
.../apptoflow/AppToFlowColumnFamily.java | 2 +-
.../storage/apptoflow/AppToFlowTable.java | 4 +-
.../storage/entity/EntityRowKey.java | 38 +++++-----
.../storage/entity/EntityTable.java | 2 +-
.../storage/flow/FlowActivityRowKey.java | 30 ++++----
.../storage/flow/FlowActivityTable.java | 2 +-
.../storage/flow/FlowRunRowKey.java | 30 ++++----
.../storage/flow/FlowRunTable.java | 2 +-
.../reader/TestTimelineReaderWebServices.java | 2 +-
...stTimelineReaderWebServicesHBaseStorage.java | 4 +-
.../storage/TestHBaseTimelineStorage.java | 4 +-
.../flow/TestHBaseStorageFlowActivity.java | 6 +-
30 files changed, 193 insertions(+), 193 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 1896c7b..f7283ae 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -245,7 +245,7 @@ public class TestMRTimelineEventHandling {
Assert.assertTrue(tmpRootFolder.isDirectory());
String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
UserGroupInformation.getCurrentUser().getShortUserName() +
- "/" + TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+ "/" + TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId) +
"/1/1/" + appId.toString();
// for this test, we expect MAPREDUCE_JOB and MAPREDUCE_TASK dirs
String outputDirJob = basePath + "/MAPREDUCE_JOB/";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index bdf6e2b..e35a32e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -493,7 +493,7 @@ public class TestDistributedShell {
YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
UserGroupInformation.getCurrentUser().getShortUserName() +
(defaultFlow ? "/" +
- TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId) +
+ TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId) +
"/1/1/" : "/test_flow_name/test_flow_version/12345678/") +
appId.toString();
// for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index 35a33b0..150026e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -160,7 +160,7 @@ public class TimelineUtils {
return SecurityUtil.buildTokenService(timelineServiceAddr);
}
- public static String generateDefaultFlowIdBasedOnAppId(ApplicationId appId) {
+ public static String generateDefaultFlowNameBasedOnAppId(ApplicationId appId) {
return "flow_" + appId.getClusterTimestamp() + "_" + appId.getId();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 57258d5..9100366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -307,7 +307,7 @@ public class TestSystemMetricsPublisherForV2 {
+ "/"
+ app.getUser()
+ "/"
- + TimelineUtils.generateDefaultFlowIdBasedOnAppId(app
+ + TimelineUtils.generateDefaultFlowNameBasedOnAppId(app
.getApplicationId()) + "/" + DEFAULT_FLOW_VERSION + "/"
+ DEFAULT_FLOW_RUN + "/" + app.getApplicationId();
return outputDirApp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index fa32211..b148491 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -55,7 +55,7 @@ public class AppLevelTimelineCollector extends TimelineCollector {
// Current user usually is not the app user, but keep this field non-null
context.setUserId(UserGroupInformation.getCurrentUser().getShortUserName());
// Use app ID to generate a default flow name for orphan app
- context.setFlowName(TimelineUtils.generateDefaultFlowIdBasedOnAppId(appId));
+ context.setFlowName(TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId));
// Set the flow version to string 1 if it's an orphan app
context.setFlowVersion("1");
// Set the flow run ID to 1 if it's an orphan app
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 294b05b..37f1671 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -66,7 +66,7 @@ public class TimelineReaderManager extends AbstractService {
* @see TimelineReader#getEntities
*/
Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -74,7 +74,7 @@ public class TimelineReaderManager extends AbstractService {
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) throws IOException {
String cluster = getClusterID(clusterId, getConfig());
- return reader.getEntities(userId, cluster, flowId, flowRunId, appId,
+ return reader.getEntities(userId, cluster, flowName, flowRunId, appId,
entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
metricFilters, eventFilters, null, null, fieldsToRetrieve);
@@ -87,10 +87,10 @@ public class TimelineReaderManager extends AbstractService {
* @see TimelineReader#getEntity
*/
public TimelineEntity getEntity(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, EnumSet<Field> fields) throws IOException {
String cluster = getClusterID(clusterId, getConfig());
- return reader.getEntity(userId, cluster, flowId, flowRunId, appId,
+ return reader.getEntity(userId, cluster, flowName, flowRunId, appId,
entityType, entityId, null, null, fields);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index d3ff8b7..f0283a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -326,7 +326,7 @@ public class TimelineReaderWebServices {
@PathParam("appid") String appId,
@PathParam("entitytype") String entityType,
@QueryParam("userid") String userId,
- @QueryParam("flowid") String flowId,
+ @QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@@ -340,7 +340,7 @@ public class TimelineReaderWebServices {
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
- return getEntities(req, res, null, appId, entityType, userId, flowId,
+ return getEntities(req, res, null, appId, entityType, userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
metricfilters, eventfilters, fields);
@@ -359,7 +359,7 @@ public class TimelineReaderWebServices {
@PathParam("appid") String appId,
@PathParam("entitytype") String entityType,
@QueryParam("userid") String userId,
- @QueryParam("flowid") String flowId,
+ @QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@@ -384,7 +384,7 @@ public class TimelineReaderWebServices {
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
- parseStr(userId), parseStr(clusterId), parseStr(flowId),
+ parseStr(userId), parseStr(clusterId), parseStr(flowName),
parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
parseLongStr(limit), parseLongStr(createdTimeStart),
parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart),
@@ -423,11 +423,11 @@ public class TimelineReaderWebServices {
@PathParam("entitytype") String entityType,
@PathParam("entityid") String entityId,
@QueryParam("userid") String userId,
- @QueryParam("flowid") String flowId,
+ @QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
return getEntity(req, res, null, appId, entityType, entityId, userId,
- flowId, flowRunId, fields);
+ flowName, flowRunId, fields);
}
/**
@@ -444,7 +444,7 @@ public class TimelineReaderWebServices {
@PathParam("entitytype") String entityType,
@PathParam("entityid") String entityId,
@QueryParam("userid") String userId,
- @QueryParam("flowid") String flowId,
+ @QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
@@ -458,7 +458,7 @@ public class TimelineReaderWebServices {
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
- parseStr(userId), parseStr(clusterId), parseStr(flowId),
+ parseStr(userId), parseStr(clusterId), parseStr(flowName),
parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
parseStr(entityId), parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
@@ -481,30 +481,30 @@ public class TimelineReaderWebServices {
* Cluster ID is not provided by client so default cluster ID has to be taken.
*/
@GET
- @Path("/flowrun/{userid}/{flowid}/{flowrunid}/")
+ @Path("/flowrun/{userid}/{flowname}/{flowrunid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getFlowRun(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@PathParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
- return getFlowRun(req, res, userId, null, flowId, flowRunId, fields);
+ return getFlowRun(req, res, userId, null, flowName, flowRunId, fields);
}
/**
* Return a single flow run for the given user, cluster, flow id and run id.
*/
@GET
- @Path("/flowrun/{userid}/{clusterid}/{flowid}/{flowrunid}/")
+ @Path("/flowrun/{userid}/{clusterid}/{flowname}/{flowrunid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getFlowRun(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@PathParam("flowrunid") String flowRunId,
@QueryParam("fields") String fields) {
String url = req.getRequestURI() +
@@ -518,7 +518,7 @@ public class TimelineReaderWebServices {
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(parseStr(userId),
- parseStr(clusterId), parseStr(flowId), parseLongStr(flowRunId), null,
+ parseStr(clusterId), parseStr(flowName), parseLongStr(flowRunId), null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null,
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
@@ -528,7 +528,7 @@ public class TimelineReaderWebServices {
if (entity == null) {
LOG.info("Processed URL " + url + " but flowrun not found (Took " +
(endTime - startTime) + " ms.)");
- throw new NotFoundException("Flow run {flow id: " + parseStr(flowId) +
+ throw new NotFoundException("Flow run {flow id: " + parseStr(flowName) +
", run id: " + parseLongStr(flowRunId) + " } is not found");
}
LOG.info("Processed URL " + url +
@@ -541,18 +541,18 @@ public class TimelineReaderWebServices {
* Cluster ID is not provided by client so default cluster ID has to be taken.
*/
@GET
- @Path("/flowruns/{userid}/{flowid}/")
+ @Path("/flowruns/{userid}/{flowname}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRuns(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
@QueryParam("fields") String fields) {
- return getFlowRuns(req, res, userId, null, flowId, limit, createdTimeStart,
+ return getFlowRuns(req, res, userId, null, flowName, limit, createdTimeStart,
createdTimeEnd, fields);
}
@@ -560,14 +560,14 @@ public class TimelineReaderWebServices {
* Return a set of flow runs for the given user, cluster and flow id.
*/
@GET
- @Path("/flowruns/{userid}/{clusterid}/{flowid}/")
+ @Path("/flowruns/{userid}/{clusterid}/{flowname}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRuns(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
@@ -583,7 +583,7 @@ public class TimelineReaderWebServices {
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
- parseStr(userId), parseStr(clusterId), parseStr(flowId), null, null,
+ parseStr(userId), parseStr(clusterId), parseStr(flowName), null, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), parseLongStr(limit),
parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), null,
null, null, null, null, null, null, null,
@@ -692,11 +692,11 @@ public class TimelineReaderWebServices {
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("appid") String appId,
- @QueryParam("flowid") String flowId,
+ @QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("userid") String userId,
@QueryParam("fields") String fields) {
- return getApp(req, res, null, appId, flowId, flowRunId, userId, fields);
+ return getApp(req, res, null, appId, flowName, flowRunId, userId, fields);
}
/**
@@ -710,7 +710,7 @@ public class TimelineReaderWebServices {
@Context HttpServletResponse res,
@PathParam("clusterid") String clusterId,
@PathParam("appid") String appId,
- @QueryParam("flowid") String flowId,
+ @QueryParam("flowname") String flowName,
@QueryParam("flowrunid") String flowRunId,
@QueryParam("userid") String userId,
@QueryParam("fields") String fields) {
@@ -725,7 +725,7 @@ public class TimelineReaderWebServices {
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(parseStr(userId),
- parseStr(clusterId), parseStr(flowId), parseLongStr(flowRunId),
+ parseStr(clusterId), parseStr(flowName), parseLongStr(flowRunId),
parseStr(appId), TimelineEntityType.YARN_APPLICATION.toString(), null,
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
@@ -749,13 +749,13 @@ public class TimelineReaderWebServices {
* is reached, will be returned.
*/
@GET
- @Path("/flowrunapps/{userid}/{flowid}/{flowrunid}/")
+ @Path("/flowrunapps/{userid}/{flowname}/{flowrunid}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRunApps(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@PathParam("flowrunid") String flowRunId,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@@ -770,7 +770,7 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
- TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
metricfilters, eventfilters, fields);
@@ -782,14 +782,14 @@ public class TimelineReaderWebServices {
* till the limit is reached, will be returned.
*/
@GET
- @Path("/flowrunapps/{userid}/{clusterid}/{flowid}/{flowrunid}/")
+ @Path("/flowrunapps/{userid}/{clusterid}/{flowname}/{flowrunid}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRunApps(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@PathParam("flowrunid") String flowRunId,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@@ -804,7 +804,7 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
- TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
metricfilters, eventfilters, fields);
@@ -817,13 +817,13 @@ public class TimelineReaderWebServices {
* reached, will be returned.
*/
@GET
- @Path("/flowapps/{userid}/{flowid}/")
+ @Path("/flowapps/{userid}/{flowname}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowApps(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
@@ -837,7 +837,7 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
- TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
metricfilters, eventfilters, fields);
@@ -849,14 +849,14 @@ public class TimelineReaderWebServices {
* is reached, will be returned.
*/
@GET
- @Path("/flowapps/{userid}/{clusterid}/{flowid}/")
+ @Path("/flowapps/{userid}/{clusterid}/{flowname}/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowApps(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
- @PathParam("flowid") String flowId,
+ @PathParam("flowname") String flowName,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
@@ -870,7 +870,7 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
- TimelineEntityType.YARN_APPLICATION.toString(), userId, flowId,
+ TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
metricfilters, eventfilters, fields);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
index 7082a5e..d812a6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
@@ -60,7 +60,7 @@ class ApplicationEntityReader extends GenericEntityReader {
new ApplicationTable();
public ApplicationEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -68,7 +68,7 @@ class ApplicationEntityReader extends GenericEntityReader {
Set<String> metricFilters, Set<String> eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
@@ -76,10 +76,10 @@ class ApplicationEntityReader extends GenericEntityReader {
}
public ApplicationEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, TimelineFilterList confsToRetrieve,
TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
}
@@ -173,7 +173,7 @@ class ApplicationEntityReader extends GenericEntityReader {
protected Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
byte[] rowKey =
- ApplicationRowKey.getRowKey(clusterId, userId, flowId, flowRunId,
+ ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
appId);
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
@@ -191,7 +191,7 @@ class ApplicationEntityReader extends GenericEntityReader {
Preconditions.checkNotNull(appId, "appId shouldn't be null");
} else {
Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(flowId, "flowId shouldn't be null");
+ Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
}
}
@@ -199,10 +199,10 @@ class ApplicationEntityReader extends GenericEntityReader {
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
if (singleEntityRead) {
- if (flowId == null || flowRunId == null || userId == null) {
+ if (flowName == null || flowRunId == null || userId == null) {
FlowContext context =
lookupFlowContext(clusterId, appId, hbaseConf, conn);
- flowId = context.flowId;
+ flowName = context.flowName;
flowRunId = context.flowRunId;
userId = context.userId;
}
@@ -244,10 +244,10 @@ class ApplicationEntityReader extends GenericEntityReader {
Scan scan = new Scan();
if (flowRunId != null) {
scan.setRowPrefixFilter(ApplicationRowKey.
- getRowKeyPrefix(clusterId, userId, flowId, flowRunId));
+ getRowKeyPrefix(clusterId, userId, flowName, flowRunId));
} else {
scan.setRowPrefixFilter(ApplicationRowKey.
- getRowKeyPrefix(clusterId, userId, flowId));
+ getRowKeyPrefix(clusterId, userId, flowName));
}
FilterList newList = new FilterList();
newList.addFilter(new PageFilter(limit));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 48bf844..36b3f0d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -150,11 +150,11 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
}
- private String getFlowRunPath(String userId, String clusterId, String flowId,
+ private String getFlowRunPath(String userId, String clusterId, String flowName,
Long flowRunId, String appId)
throws IOException {
- if (userId != null && flowId != null && flowRunId != null) {
- return userId + "/" + flowId + "/" + flowRunId;
+ if (userId != null && flowName != null && flowRunId != null) {
+ return userId + "/" + flowName + "/" + flowRunId;
}
if (clusterId == null || appId == null) {
throw new IOException("Unable to get flow info");
@@ -387,11 +387,11 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@Override
public TimelineEntity getEntity(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, TimelineFilterList confsToRetrieve,
TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve)
throws IOException {
- String flowRunPath = getFlowRunPath(userId, clusterId, flowId,
+ String flowRunPath = getFlowRunPath(userId, clusterId, flowName,
flowRunId, appId);
File dir = new File(new File(rootPath, ENTITIES_DIR),
clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
@@ -411,7 +411,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
@Override
public Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -420,7 +420,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve) throws IOException {
String flowRunPath =
- getFlowRunPath(userId, clusterId, flowId, flowRunId, appId);
+ getFlowRunPath(userId, clusterId, flowName, flowRunId, appId);
File dir =
new File(new File(rootPath, ENTITIES_DIR),
clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
index 71dd0a1..7e8d4ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
@@ -49,23 +49,23 @@ class FlowActivityEntityReader extends TimelineEntityReader {
new FlowActivityTable();
public FlowActivityEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
eventFilters, null, null, fieldsToRetrieve, true);
}
public FlowActivityEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
null, null, fieldsToRetrieve);
}
@@ -135,7 +135,7 @@ class FlowActivityEntityReader extends TimelineEntityReader {
long time = rowKey.getDayTimestamp();
String user = rowKey.getUserId();
- String flowName = rowKey.getFlowId();
+ String flowName = rowKey.getFlowName();
FlowActivityEntity flowActivity =
new FlowActivityEntity(clusterId, time, user, flowName);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
index 1895fa6..c9076ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
@@ -58,7 +58,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
private static final FlowRunTable FLOW_RUN_TABLE = new FlowRunTable();
public FlowRunEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -66,17 +66,17 @@ class FlowRunEntityReader extends TimelineEntityReader {
Set<String> metricFilters, Set<String> eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
eventFilters, null, metricsToRetrieve, fieldsToRetrieve, true);
}
public FlowRunEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, TimelineFilterList confsToRetrieve,
TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
null, metricsToRetrieve, fieldsToRetrieve);
}
@@ -92,7 +92,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
protected void validateParams() {
Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(flowId, "flowId shouldn't be null");
+ Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
if (singleEntityRead) {
Preconditions.checkNotNull(flowRunId, "flowRunId shouldn't be null");
}
@@ -155,7 +155,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
protected Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
byte[] rowKey =
- FlowRunRowKey.getRowKey(clusterId, userId, flowId, flowRunId);
+ FlowRunRowKey.getRowKey(clusterId, userId, flowName, flowRunId);
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
if (filterList != null && !filterList.getFilters().isEmpty()) {
@@ -169,7 +169,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
Connection conn, FilterList filterList) throws IOException {
Scan scan = new Scan();
scan.setRowPrefixFilter(
- FlowRunRowKey.getRowKeyPrefix(clusterId, userId, flowId));
+ FlowRunRowKey.getRowKeyPrefix(clusterId, userId, flowName));
FilterList newList = new FilterList();
newList.addFilter(new PageFilter(limit));
if (filterList != null && !filterList.getFilters().isEmpty()) {
@@ -183,7 +183,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
protected TimelineEntity parseEntity(Result result) throws IOException {
FlowRunEntity flowRun = new FlowRunEntity();
flowRun.setUser(userId);
- flowRun.setName(flowId);
+ flowRun.setName(flowName);
if (singleEntityRead) {
flowRun.setRunId(flowRunId);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
index dcb8b89..784dfd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
@@ -76,7 +76,7 @@ class GenericEntityReader extends TimelineEntityReader {
private final AppToFlowTable appToFlowTable = new AppToFlowTable();
public GenericEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -84,7 +84,7 @@ class GenericEntityReader extends TimelineEntityReader {
Set<String> metricFilters, Set<String> eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, limit,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
@@ -92,10 +92,10 @@ class GenericEntityReader extends TimelineEntityReader {
}
public GenericEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, TimelineFilterList confsToRetrieve,
TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowId, flowRunId, appId, entityType, entityId,
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
}
@@ -204,11 +204,11 @@ class GenericEntityReader extends TimelineEntityReader {
protected static class FlowContext {
protected final String userId;
- protected final String flowId;
+ protected final String flowName;
protected final Long flowRunId;
- public FlowContext(String user, String flowId, Long flowRunId) {
+ public FlowContext(String user, String flowName, Long flowRunId) {
this.userId = user;
- this.flowId = flowId;
+ this.flowName = flowName;
this.flowRunId = flowRunId;
}
}
@@ -227,10 +227,10 @@ class GenericEntityReader extends TimelineEntityReader {
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
// In reality all three should be null or neither should be null
- if (flowId == null || flowRunId == null || userId == null) {
+ if (flowName == null || flowRunId == null || userId == null) {
FlowContext context =
lookupFlowContext(clusterId, appId, hbaseConf, conn);
- flowId = context.flowId;
+ flowName = context.flowName;
flowRunId = context.flowRunId;
userId = context.userId;
}
@@ -269,7 +269,7 @@ class GenericEntityReader extends TimelineEntityReader {
protected Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
byte[] rowKey =
- EntityRowKey.getRowKey(clusterId, userId, flowId, flowRunId, appId,
+ EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
entityType, entityId);
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
@@ -286,7 +286,7 @@ class GenericEntityReader extends TimelineEntityReader {
// and one type
Scan scan = new Scan();
scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
- clusterId, userId, flowId, flowRunId, appId, entityType));
+ clusterId, userId, flowName, flowRunId, appId, entityType));
scan.setMaxVersions(Integer.MAX_VALUE);
if (filterList != null && !filterList.getFilters().isEmpty()) {
scan.setFilter(filterList);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 9e4b26a..96c5a19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -64,20 +64,20 @@ public class HBaseTimelineReaderImpl
@Override
public TimelineEntity getEntity(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, TimelineFilterList confsToRetrieve,
TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve)
throws IOException {
TimelineEntityReader reader =
TimelineEntityReaderFactory.createSingleEntityReader(userId, clusterId,
- flowId, flowRunId, appId, entityType, entityId, confsToRetrieve,
+ flowName, flowRunId, appId, entityType, entityId, confsToRetrieve,
metricsToRetrieve, fieldsToRetrieve);
return reader.readEntity(hbaseConf, conn);
}
@Override
public Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -87,7 +87,7 @@ public class HBaseTimelineReaderImpl
EnumSet<Field> fieldsToRetrieve) throws IOException {
TimelineEntityReader reader =
TimelineEntityReaderFactory.createMultipleEntitiesReader(userId,
- clusterId, flowId, flowRunId, appId, entityType, limit,
+ clusterId, flowName, flowRunId, appId, entityType, limit,
createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
metricFilters, eventFilters, confsToRetrieve, metricsToRetrieve,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
index 7178aab..a26c0c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
@@ -53,7 +53,7 @@ abstract class TimelineEntityReader {
protected String userId;
protected String clusterId;
- protected String flowId;
+ protected String flowName;
protected Long flowRunId;
protected String appId;
protected String entityType;
@@ -92,7 +92,7 @@ abstract class TimelineEntityReader {
* Instantiates a reader for multiple-entity reads.
*/
protected TimelineEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -104,7 +104,7 @@ abstract class TimelineEntityReader {
this.sortedKeys = sortedKeys;
this.userId = userId;
this.clusterId = clusterId;
- this.flowId = flowId;
+ this.flowName = flowName;
this.flowRunId = flowRunId;
this.appId = appId;
this.entityType = entityType;
@@ -130,13 +130,13 @@ abstract class TimelineEntityReader {
* Instantiates a reader for single-entity reads.
*/
protected TimelineEntityReader(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId, TimelineFilterList confsToRetrieve,
TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
this.singleEntityRead = true;
this.userId = userId;
this.clusterId = clusterId;
- this.flowId = flowId;
+ this.flowName = flowName;
this.flowRunId = flowRunId;
this.appId = appId;
this.entityType = entityType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
index 16204c5..36ed4ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
@@ -34,23 +34,23 @@ class TimelineEntityReaderFactory {
* the specified input.
*/
public static TimelineEntityReader createSingleEntityReader(String userId,
- String clusterId, String flowId, Long flowRunId, String appId,
+ String clusterId, String flowName, Long flowRunId, String appId,
String entityType, String entityId, TimelineFilterList confs,
TimelineFilterList metrics, EnumSet<Field> fieldsToRetrieve) {
// currently the types that are handled separate from the generic entity
// table are application, flow run, and flow activity entities
if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
- return new ApplicationEntityReader(userId, clusterId, flowId, flowRunId,
+ return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
- return new FlowRunEntityReader(userId, clusterId, flowId, flowRunId,
+ return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
- return new FlowActivityEntityReader(userId, clusterId, flowId, flowRunId,
+ return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, entityId, fieldsToRetrieve);
} else {
// assume we're dealing with a generic entity read
- return new GenericEntityReader(userId, clusterId, flowId, flowRunId,
+ return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
}
}
@@ -60,7 +60,7 @@ class TimelineEntityReaderFactory {
* the specified input and predicates.
*/
public static TimelineEntityReader createMultipleEntitiesReader(String userId,
- String clusterId, String flowId, Long flowRunId, String appId,
+ String clusterId, String flowName, Long flowRunId, String appId,
String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -71,26 +71,26 @@ class TimelineEntityReaderFactory {
// currently the types that are handled separate from the generic entity
// table are application, flow run, and flow activity entities
if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
- return new ApplicationEntityReader(userId, clusterId, flowId, flowRunId,
+ return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, limit, createdTimeBegin, createdTimeEnd,
modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
infoFilters, configFilters, metricFilters, eventFilters, confs,
metrics, fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
- return new FlowActivityEntityReader(userId, clusterId, flowId, flowRunId,
+ return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, limit, createdTimeBegin, createdTimeEnd,
modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
infoFilters, configFilters, metricFilters, eventFilters,
fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
- return new FlowRunEntityReader(userId, clusterId, flowId, flowRunId,
+ return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, limit, createdTimeBegin, createdTimeEnd,
modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
infoFilters, configFilters, metricFilters, eventFilters, confs,
metrics, fieldsToRetrieve);
} else {
// assume we're dealing with a generic entity read
- return new GenericEntityReader(userId, clusterId, flowId, flowRunId,
+ return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
appId, entityType, limit, createdTimeBegin, createdTimeEnd,
modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
infoFilters, configFilters, metricFilters, eventFilters, confs,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index 0ed17da..56bd3a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -62,7 +62,7 @@ public interface TimelineReader extends Service {
* Context user Id(optional).
* @param clusterId
* Context cluster Id(mandatory).
- * @param flowId
+ * @param flowName
* Context flow Id (optional).
* @param flowRunId
* Context flow run Id (optional).
@@ -93,7 +93,7 @@ public interface TimelineReader extends Service {
* contain the metadata plus the given fields to retrieve.
* @throws IOException
*/
- TimelineEntity getEntity(String userId, String clusterId, String flowId,
+ TimelineEntity getEntity(String userId, String clusterId, String flowName,
Long flowRunId, String appId, String entityType, String entityId,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve) throws IOException;
@@ -113,7 +113,7 @@ public interface TimelineReader extends Service {
* Context user Id(optional).
* @param clusterId
* Context cluster Id(mandatory).
- * @param flowId
+ * @param flowName
* Context flow Id (optional).
* @param flowRunId
* Context flow run Id (optional).
@@ -183,7 +183,7 @@ public interface TimelineReader extends Service {
* @throws IOException
*/
Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
index 1cf6145..3c182b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -27,15 +27,15 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStor
public class ApplicationRowKey {
private final String clusterId;
private final String userId;
- private final String flowId;
+ private final String flowName;
private final long flowRunId;
private final String appId;
- public ApplicationRowKey(String clusterId, String userId, String flowId,
+ public ApplicationRowKey(String clusterId, String userId, String flowName,
long flowRunId, String appId) {
this.clusterId = clusterId;
this.userId = userId;
- this.flowId = flowId;
+ this.flowName = flowName;
this.flowRunId = flowRunId;
this.appId = appId;
}
@@ -48,8 +48,8 @@ public class ApplicationRowKey {
return userId;
}
- public String getFlowId() {
- return flowId;
+ public String getFlowName() {
+ return flowName;
}
public long getFlowRunId() {
@@ -62,54 +62,54 @@ public class ApplicationRowKey {
/**
* Constructs a row key prefix for the application table as follows:
- * {@code clusterId!userName!flowId!}
+ * {@code clusterId!userName!flowName!}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
- String flowId) {
+ String flowName) {
byte[] first = Bytes.toBytes(
- Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowId));
+ Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowName));
return Separator.QUALIFIERS.join(first, new byte[0]);
}
/**
* Constructs a row key prefix for the application table as follows:
- * {@code clusterId!userName!flowId!flowRunId!}
+ * {@code clusterId!userName!flowName!flowRunId!}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @param flowRunId
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
- String flowId, Long flowRunId) {
+ String flowName, Long flowRunId) {
byte[] first = Bytes.toBytes(
- Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowId));
+ Separator.QUALIFIERS.joinEncoded(clusterId, userId, flowName));
byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
return Separator.QUALIFIERS.join(first, second, new byte[0]);
}
/**
* Constructs a row key for the application table as follows:
- * {@code clusterId!userName!flowId!flowRunId!AppId}
+ * {@code clusterId!userName!flowName!flowRunId!AppId}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @param flowRunId
* @param appId
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String userId,
- String flowId, Long flowRunId, String appId) {
+ String flowName, Long flowRunId, String appId) {
byte[] first =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, userId,
- flowId));
+ flowName));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
@@ -132,11 +132,11 @@ public class ApplicationRowKey {
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
String userId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
- String flowId =
+ String flowName =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
long flowRunId =
TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
String appId = TimelineStorageUtils.decodeAppId(rowKeyComponents[4]);
- return new ApplicationRowKey(clusterId, userId, flowId, flowRunId, appId);
+ return new ApplicationRowKey(clusterId, userId, flowName, flowRunId, appId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index a997997..ee5ff83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* |-------------------------------------------------------------------------|
* | clusterId! | id:appId | metricId1: | configKey1: |
* | userName! | | metricValue1 | configValue1 |
- * | flowId! | created_time: | @timestamp1 | |
+ * | flowName! | created_time: | @timestamp1 | |
* | flowRunId! | 1392993084018 | | configKey2: |
* | AppId | | metriciD1: | configValue2 |
* | | modified_time: | metricValue2 | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
index e74235f..d722d15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
*/
public enum AppToFlowColumnFamily implements ColumnFamily<AppToFlowTable> {
/**
- * Mapping column family houses known columns such as flowId and flowRunId
+ * Mapping column family houses known columns such as flowName and flowRunId
*/
MAPPING("m");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
index b30f253..868c6f3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
@@ -34,7 +34,7 @@ import java.io.IOException;
/**
* The app_flow table as column families mapping. Mapping stores
- * appId to flowId and flowRunId mapping information
+ * appId to flowName and flowRunId mapping information
*
* Example app_flow table record:
*
@@ -43,7 +43,7 @@ import java.io.IOException;
* | Row | Column Family |
* | key | info |
* |--------------------------------------|
- * | clusterId! | flowId: |
+ * | clusterId! | flowName: |
* | AppId | foo@daily_hive_report |
* | | |
* | | flowRunId: |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index e0413c0..3eaa78c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -27,17 +27,17 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStor
public class EntityRowKey {
private final String clusterId;
private final String userId;
- private final String flowId;
+ private final String flowName;
private final long flowRunId;
private final String appId;
private final String entityType;
private final String entityId;
- public EntityRowKey(String clusterId, String userId, String flowId,
+ public EntityRowKey(String clusterId, String userId, String flowName,
long flowRunId, String appId, String entityType, String entityId) {
this.clusterId = clusterId;
this.userId = userId;
- this.flowId = flowId;
+ this.flowName = flowName;
this.flowRunId = flowRunId;
this.appId = appId;
this.entityType = entityType;
@@ -52,8 +52,8 @@ public class EntityRowKey {
return userId;
}
- public String getFlowId() {
- return flowId;
+ public String getFlowName() {
+ return flowName;
}
public long getFlowRunId() {
@@ -74,20 +74,20 @@ public class EntityRowKey {
/**
* Constructs a row key prefix for the entity table as follows:
- * {@code userName!clusterId!flowId!flowRunId!AppId}
+ * {@code userName!clusterId!flowName!flowRunId!AppId}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @param flowRunId
* @param appId
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
- String flowId, Long flowRunId, String appId) {
+ String flowName, Long flowRunId, String appId) {
byte[] first =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
- flowId));
+ flowName));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
@@ -97,21 +97,21 @@ public class EntityRowKey {
/**
* Constructs a row key prefix for the entity table as follows:
- * {@code userName!clusterId!flowId!flowRunId!AppId!entityType!}
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @param flowRunId
* @param appId
* @param entityType
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
- String flowId, Long flowRunId, String appId, String entityType) {
+ String flowName, Long flowRunId, String appId, String entityType) {
byte[] first =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
- flowId));
+ flowName));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
@@ -123,11 +123,11 @@ public class EntityRowKey {
/**
* Constructs a row key for the entity table as follows:
- * {@code userName!clusterId!flowId!flowRunId!AppId!entityType!entityId}
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @param flowRunId
* @param appId
* @param entityType
@@ -135,11 +135,11 @@ public class EntityRowKey {
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String userId,
- String flowId, Long flowRunId, String appId, String entityType,
+ String flowName, Long flowRunId, String appId, String entityType,
String entityId) {
byte[] first =
Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(userId, clusterId,
- flowId));
+ flowName));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
@@ -164,7 +164,7 @@ public class EntityRowKey {
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
String clusterId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
- String flowId =
+ String flowName =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
long flowRunId =
TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
@@ -173,7 +173,7 @@ public class EntityRowKey {
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[5]));
String entityId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[6]));
- return new EntityRowKey(clusterId, userId, flowId, flowRunId, appId,
+ return new EntityRowKey(clusterId, userId, flowName, flowRunId, appId,
entityType, entityId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 96773b2..15d4465 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -46,7 +46,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* |-------------------------------------------------------------------------|
* | userName! | id:entityId | metricId1: | configKey1: |
* | clusterId! | | metricValue1 | configValue1 |
- * | flowId! | type:entityType | @timestamp1 | |
+ * | flowName! | type:entityType | @timestamp1 | |
* | flowRunId! | | | configKey2: |
* | AppId! | created_time: | metriciD1: | configValue2 |
* | entityType!| 1392993084018 | metricValue2 | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
index fc1aa70..a9598ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -29,14 +29,14 @@ public class FlowActivityRowKey {
private final String clusterId;
private final long dayTs;
private final String userId;
- private final String flowId;
+ private final String flowName;
public FlowActivityRowKey(String clusterId, long dayTs, String userId,
- String flowId) {
+ String flowName) {
this.clusterId = clusterId;
this.dayTs = dayTs;
this.userId = userId;
- this.flowId = flowId;
+ this.flowName = flowName;
}
public String getClusterId() {
@@ -51,8 +51,8 @@ public class FlowActivityRowKey {
return userId;
}
- public String getFlowId() {
- return flowId;
+ public String getFlowName() {
+ return flowName;
}
/**
@@ -82,38 +82,38 @@ public class FlowActivityRowKey {
/**
* Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowId}
+ * {@code clusterId!dayTimestamp!user!flowName}
*
* Will insert into current day's record in the table
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @return byte array with the row key prefix
*/
public static byte[] getRowKey(String clusterId, String userId,
- String flowId) {
+ String flowName) {
long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
- return getRowKey(clusterId, dayTs, userId, flowId);
+ return getRowKey(clusterId, dayTs, userId, flowName);
}
/**
* Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowId}
+ * {@code clusterId!dayTimestamp!user!flowName}
*
* @param clusterId
* @param dayTs
* @param userId
- * @param flowId
+ * @param flowName
* @return byte array for the row key
*/
public static byte[] getRowKey(String clusterId, long dayTs, String userId,
- String flowId) {
+ String flowName) {
return Separator.QUALIFIERS.join(
Bytes.toBytes(Separator.QUALIFIERS.encode(clusterId)),
Bytes.toBytes(TimelineStorageUtils.invertLong(dayTs)),
Bytes.toBytes(Separator.QUALIFIERS.encode(userId)),
- Bytes.toBytes(Separator.QUALIFIERS.encode(flowId)));
+ Bytes.toBytes(Separator.QUALIFIERS.encode(flowName)));
}
/**
@@ -133,8 +133,8 @@ public class FlowActivityRowKey {
TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[1]));
String userId = Separator.QUALIFIERS.decode(Bytes
.toString(rowKeyComponents[2]));
- String flowId = Separator.QUALIFIERS.decode(Bytes
+ String flowName = Separator.QUALIFIERS.decode(Bytes
.toString(rowKeyComponents[3]));
- return new FlowActivityRowKey(clusterId, dayTs, userId, flowId);
+ return new FlowActivityRowKey(clusterId, dayTs, userId, flowName);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
index af8df99..315281f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
* | inv Top of | |
* | Day! | r!runid2:version7 |
* | userName! | |
- * | flowId | |
+ * | flowName | |
* |-------------------------------------------|
* </pre>
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: YARN-4238. createdTime and
modifiedTime is not reported while publishing entities to ATSv2. (Varun
Saxena via Naganarasimha G R)
Posted by gt...@apache.org.
YARN-4238. createdTime and modifiedTime is not reported while publishing entities to ATSv2. (Varun Saxena via Naganarasimha G R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd369a54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd369a54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd369a54
Branch: refs/heads/YARN-2928
Commit: fd369a54e3cc08b812746f7c339826b0c4c7977f
Parents: 2269e78
Author: Naganarasimha <na...@apache.com>
Authored: Wed Jan 27 11:59:40 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:24:12 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 47 ++++++----
.../records/timelineservice/TimelineEntity.java | 19 ----
.../TestTimelineServiceRecords.java | 1 -
.../monitor/ContainersMonitorImpl.java | 6 +-
.../timelineservice/NMTimelinePublisher.java | 6 +-
.../TestSystemMetricsPublisherForV2.java | 34 +++++--
.../reader/TimelineReaderManager.java | 7 +-
.../reader/TimelineReaderWebServices.java | 51 ++++-------
.../storage/FileSystemTimelineReaderImpl.java | 21 +----
.../storage/HBaseTimelineReaderImpl.java | 8 +-
.../storage/HBaseTimelineWriterImpl.java | 4 -
.../PhoenixOfflineAggregationWriterImpl.java | 9 +-
.../timelineservice/storage/TimelineReader.java | 32 +++----
.../storage/application/ApplicationColumn.java | 5 --
.../storage/application/ApplicationTable.java | 11 +--
.../storage/entity/EntityColumn.java | 5 --
.../storage/entity/EntityTable.java | 11 +--
.../storage/reader/ApplicationEntityReader.java | 23 +----
.../reader/FlowActivityEntityReader.java | 7 +-
.../storage/reader/FlowRunEntityReader.java | 7 +-
.../storage/reader/GenericEntityReader.java | 22 +----
.../storage/reader/TimelineEntityReader.java | 5 --
.../reader/TimelineEntityReaderFactory.java | 27 +++---
.../reader/TestTimelineReaderWebServices.java | 48 +---------
.../TestFileSystemTimelineReaderImpl.java | 92 +++++--------------
.../TestFileSystemTimelineWriterImpl.java | 3 +-
.../storage/TestHBaseTimelineStorage.java | 95 +++++++-------------
...TestPhoenixOfflineAggregationWriterImpl.java | 1 -
.../storage/flow/TestFlowDataGenerator.java | 1 -
.../flow/TestHBaseStorageFlowActivity.java | 6 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 9 +-
31 files changed, 197 insertions(+), 426 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index d88588c..a10872a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1135,10 +1135,10 @@ public class JobHistoryEventHandler extends AbstractService
// jobId, timestamp and entityType.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createJobEntity(HistoryEvent event, long timestamp, JobId jobId,
- String entityType) {
-
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
- createBaseEntity(event, timestamp, entityType);
+ String entityType, boolean setCreatedTime) {
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(jobId.toString());
return entity;
}
@@ -1146,8 +1146,9 @@ public class JobHistoryEventHandler extends AbstractService
// create BaseEntity from HistoryEvent with adding other info, like:
// timestamp and entityType.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
- createBaseEntity(HistoryEvent event, long timestamp, String entityType) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent =
+ createBaseEntity(HistoryEvent event, long timestamp, String entityType,
+ boolean setCreatedTime) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent tEvent =
event.toTimelineEvent();
tEvent.setTimestamp(timestamp);
@@ -1155,6 +1156,9 @@ public class JobHistoryEventHandler extends AbstractService
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.addEvent(tEvent);
entity.setType(entityType);
+ if (setCreatedTime) {
+ entity.setCreatedTime(timestamp);
+ }
return entity;
}
@@ -1162,9 +1166,10 @@ public class JobHistoryEventHandler extends AbstractService
// taskId, jobId, timestamp, entityType and relatedJobEntity.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createTaskEntity(HistoryEvent event, long timestamp, String taskId,
- String entityType, String relatedJobEntity, JobId jobId) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
- createBaseEntity(event, timestamp, entityType);
+ String entityType, String relatedJobEntity, JobId jobId,
+ boolean setCreatedTime) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskId);
entity.addIsRelatedToEntity(relatedJobEntity, jobId.toString());
return entity;
@@ -1175,9 +1180,9 @@ public class JobHistoryEventHandler extends AbstractService
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createTaskAttemptEntity(HistoryEvent event, long timestamp,
String taskAttemptId, String entityType, String relatedTaskEntity,
- String taskId) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
- createBaseEntity(event, timestamp, entityType);
+ String taskId, boolean setCreatedTime) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ createBaseEntity(event, timestamp, entityType, setCreatedTime);
entity.setId(taskAttemptId);
entity.addIsRelatedToEntity(relatedTaskEntity, taskId);
return entity;
@@ -1188,10 +1193,13 @@ public class JobHistoryEventHandler extends AbstractService
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null;
String taskId = null;
String taskAttemptId = null;
+ boolean setCreatedTime = false;
switch (event.getEventType()) {
// Handle job events
case JOB_SUBMITTED:
+ setCreatedTime = true;
+ break;
case JOB_STATUS_CHANGED:
case JOB_INFO_CHANGED:
case JOB_INITED:
@@ -1206,6 +1214,7 @@ public class JobHistoryEventHandler extends AbstractService
break;
// Handle task events
case TASK_STARTED:
+ setCreatedTime = true;
taskId = ((TaskStartedEvent)event).getTaskId().toString();
break;
case TASK_FAILED:
@@ -1218,8 +1227,13 @@ public class JobHistoryEventHandler extends AbstractService
taskId = ((TaskFinishedEvent)event).getTaskId().toString();
break;
case MAP_ATTEMPT_STARTED:
- case CLEANUP_ATTEMPT_STARTED:
case REDUCE_ATTEMPT_STARTED:
+ setCreatedTime = true;
+ taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptStartedEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case CLEANUP_ATTEMPT_STARTED:
case SETUP_ATTEMPT_STARTED:
taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
taskAttemptId = ((TaskAttemptStartedEvent)event).
@@ -1258,17 +1272,18 @@ public class JobHistoryEventHandler extends AbstractService
if (taskId == null) {
// JobEntity
tEntity = createJobEntity(event, timestamp, jobId,
- MAPREDUCE_JOB_ENTITY_TYPE);
+ MAPREDUCE_JOB_ENTITY_TYPE, setCreatedTime);
} else {
if (taskAttemptId == null) {
// TaskEntity
tEntity = createTaskEntity(event, timestamp, taskId,
- MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE, jobId);
+ MAPREDUCE_TASK_ENTITY_TYPE, MAPREDUCE_JOB_ENTITY_TYPE,
+ jobId, setCreatedTime);
} else {
// TaskAttemptEntity
tEntity = createTaskAttemptEntity(event, timestamp, taskAttemptId,
MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE, MAPREDUCE_TASK_ENTITY_TYPE,
- taskId);
+ taskId, setCreatedTime);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index 0701001..dcf2473 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -140,7 +140,6 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
private HashMap<String, Set<String>> isRelatedToEntities = new HashMap<>();
private HashMap<String, Set<String>> relatesToEntities = new HashMap<>();
private long createdTime;
- private long modifiedTime;
public TimelineEntity() {
identifier = new Identifier();
@@ -505,24 +504,6 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- @XmlElement(name = "modifiedtime")
- public long getModifiedTime() {
- if (real == null) {
- return modifiedTime;
- } else {
- return real.getModifiedTime();
- }
- }
-
- @JsonSetter("modifiedtime")
- public void setModifiedTime(long modifiedTime) {
- if (real == null) {
- this.modifiedTime = modifiedTime;
- } else {
- real.setModifiedTime(modifiedTime);
- }
- }
-
public boolean isValid() {
return (getId() != null && getType() != null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 7c9acf2..51ec762 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -139,7 +139,6 @@ public class TestTimelineServiceRecords {
event1, event2);
entity.setCreatedTime(0L);
- entity.setModifiedTime(1L);
entity.addRelatesToEntity("test type 2", "test id 2");
entity.addRelatesToEntity("test type 3", "test id 3");
entity.addIsRelatedToEntity("test type 4", "test id 4");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 739d363..da4aac2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -463,7 +463,6 @@ public class ContainersMonitorImpl extends AbstractService implements
pTree.updateProcessTree(); // update process-tree
long currentVmemUsage = pTree.getVirtualMemorySize();
long currentPmemUsage = pTree.getRssMemorySize();
- long currentTime = System.currentTimeMillis();
// if machine has 6 cores and 3 are used,
// cpuUsagePercentPerCore should be 300% and
@@ -568,9 +567,8 @@ public class ContainersMonitorImpl extends AbstractService implements
NMTimelinePublisher nmMetricsPublisher =
container.getNMTimelinePublisher();
if (nmMetricsPublisher != null) {
- nmMetricsPublisher.reportContainerResourceUsage(
- container, currentTime, pId, currentPmemUsage,
- cpuUsageTotalCoresPercentage);
+ nmMetricsPublisher.reportContainerResourceUsage(container, pId,
+ currentPmemUsage, cpuUsageTotalCoresPercentage);
}
} catch (Exception e) {
// Log the exception and proceed to the next container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 69de433..affaaae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -113,9 +113,8 @@ public class NMTimelinePublisher extends CompositeService {
}
@SuppressWarnings("unchecked")
- public void reportContainerResourceUsage(Container container,
- long createdTime, String pId, Long pmemUsage,
- Float cpuUsageTotalCoresPercentage) {
+ public void reportContainerResourceUsage(Container container, String pId,
+ Long pmemUsage, Float cpuUsageTotalCoresPercentage) {
if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE ||
cpuUsageTotalCoresPercentage !=
ResourceCalculatorProcessTree.UNAVAILABLE) {
@@ -164,6 +163,7 @@ public class NMTimelinePublisher extends CompositeService {
tEvent.setTimestamp(timestamp);
entity.addEvent(tEvent);
+ entity.setCreatedTime(timestamp);
putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 9100366..1a58cb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
@@ -44,10 +45,15 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.DrainDispatcher;
+import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
@@ -58,6 +64,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.AfterClass;
@@ -201,8 +208,7 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- Assert.assertEquals("Expected 3 events to be published", 3,
- getNumOfNonEmptyLines(appFile));
+ verifyEntity(appFile, 3, ApplicationMetricsConstants.CREATED_EVENT_TYPE);
}
@Test(timeout = 10000)
@@ -236,8 +242,7 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- Assert.assertEquals("Expected 2 events to be published", 2,
- getNumOfNonEmptyLines(appFile));
+ verifyEntity(appFile,2, AppAttemptMetricsConstants.REGISTERED_EVENT_TYPE);
}
@Test(timeout = 10000)
@@ -268,8 +273,8 @@ public class TestSystemMetricsPublisherForV2 {
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
File appFile = new File(outputDirApp, timelineServiceFileName);
Assert.assertTrue(appFile.exists());
- Assert.assertEquals("Expected 2 events to be published", 2,
- getNumOfNonEmptyLines(appFile));
+ verifyEntity(appFile, 2,
+ ContainerMetricsConstants.CREATED_IN_RM_EVENT_TYPE);
}
private RMApp createAppAndRegister(ApplicationId appId) {
@@ -282,20 +287,31 @@ public class TestSystemMetricsPublisherForV2 {
return app;
}
- private long getNumOfNonEmptyLines(File entityFile) throws IOException {
+ private static void verifyEntity(File entityFile, long expectedEvents,
+ String eventForCreatedTime) throws IOException {
BufferedReader reader = null;
String strLine;
long count = 0;
try {
reader = new BufferedReader(new FileReader(entityFile));
while ((strLine = reader.readLine()) != null) {
- if (strLine.trim().length() > 0)
+ if (strLine.trim().length() > 0) {
+ TimelineEntity entity = FileSystemTimelineReaderImpl.
+ getTimelineRecordFromJSON(strLine.trim(), TimelineEntity.class);
+ for (TimelineEvent event : entity.getEvents()) {
+ if (event.getId().equals(eventForCreatedTime)) {
+ assertTrue(entity.getCreatedTime() > 0);
+ break;
+ }
+ }
count++;
+ }
}
} finally {
reader.close();
}
- return count;
+ assertEquals("Expected " + expectedEvents + " events to be published",
+ count, expectedEvents);
}
private String getTimelineEntityDir(RMApp app) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 37f1671..65d251d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -68,16 +68,15 @@ public class TimelineReaderManager extends AbstractService {
Set<TimelineEntity> getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) throws IOException {
String cluster = getClusterID(clusterId, getConfig());
return reader.getEntities(userId, cluster, flowName, flowRunId, appId,
- entityType, limit, createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
- modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
- metricFilters, eventFilters, null, null, fieldsToRetrieve);
+ entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ null, null, fieldsToRetrieve);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index f0283a5..a054ee5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -331,8 +331,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -341,9 +339,9 @@ public class TimelineReaderWebServices {
@QueryParam("eventfilters") String eventfilters,
@QueryParam("fields") String fields) {
return getEntities(req, res, null, appId, entityType, userId, flowName,
- flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+ isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+ fields);
}
/**
@@ -364,8 +362,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -387,8 +383,7 @@ public class TimelineReaderWebServices {
parseStr(userId), parseStr(clusterId), parseStr(flowName),
parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
parseLongStr(limit), parseLongStr(createdTimeStart),
- parseLongStr(createdTimeEnd), parseLongStr(modifiedTimeStart),
- parseLongStr(modifiedTimeEnd),
+ parseLongStr(createdTimeEnd),
parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
@@ -398,7 +393,7 @@ public class TimelineReaderWebServices {
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime,
- "createdTime or modifiedTime start/end or limit or flowrunid");
+ "createdTime start/end or limit or flowrunid");
}
long endTime = Time.monotonicNow();
if (entities == null) {
@@ -585,8 +580,8 @@ public class TimelineReaderWebServices {
entities = timelineReaderManager.getEntities(
parseStr(userId), parseStr(clusterId), parseStr(flowName), null, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), parseLongStr(limit),
- parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd), null,
- null, null, null, null, null, null, null,
+ parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd),
+ null, null, null, null, null, null,
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
@@ -668,7 +663,7 @@ public class TimelineReaderWebServices {
null, parseStr(clusterId), null, null, null,
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), parseLongStr(limit),
range.dateStart, range.dateEnd, null, null, null, null, null, null,
- null, null, parseFieldsStr(fields, COMMA_DELIMITER));
+ parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "limit");
}
@@ -760,8 +755,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -771,9 +764,9 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+ isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+ fields);
}
/**
@@ -794,8 +787,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -805,9 +796,9 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- flowRunId, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
+ isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
+ fields);
}
/**
@@ -827,8 +818,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -838,9 +827,8 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, null, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters, fields);
}
/**
@@ -860,8 +848,6 @@ public class TimelineReaderWebServices {
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
- @QueryParam("modifiedtimestart") String modifiedTimeStart,
- @QueryParam("modifiedtimeend") String modifiedTimeEnd,
@QueryParam("relatesto") String relatesTo,
@QueryParam("isrelatedto") String isRelatedTo,
@QueryParam("infofilters") String infofilters,
@@ -871,8 +857,7 @@ public class TimelineReaderWebServices {
@QueryParam("fields") String fields) {
return getEntities(req, res, clusterId, null,
TimelineEntityType.YARN_APPLICATION.toString(), userId, flowName,
- null, limit, createdTimeStart, createdTimeEnd, modifiedTimeStart,
- modifiedTimeEnd, relatesTo, isRelatedTo, infofilters, conffilters,
- metricfilters, eventfilters, fields);
+ null, limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters, fields);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 36b3f0d..72c9249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -188,7 +188,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
TimelineEntity entityToBeReturned = new TimelineEntity();
entityToBeReturned.setIdentifier(entity.getIdentifier());
entityToBeReturned.setCreatedTime(entity.getCreatedTime());
- entityToBeReturned.setModifiedTime(entity.getModifiedTime());
if (fieldsToRetrieve != null) {
fillFields(entityToBeReturned, entity, fieldsToRetrieve);
}
@@ -206,9 +205,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (entity2.getCreatedTime() > 0) {
entity1.setCreatedTime(entity2.getCreatedTime());
}
- if (entity2.getModifiedTime() > 0) {
- entity1.setModifiedTime(entity2.getModifiedTime());
- }
for (Entry<String, String> configEntry : entity2.getConfigs().entrySet()) {
entity1.addConfig(configEntry.getKey(), configEntry.getValue());
}
@@ -268,8 +264,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
private Set<TimelineEntity> getEntities(File dir, String entityType,
- Long limit, Long createdTimeBegin,
- Long createdTimeEnd, Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
@@ -284,12 +279,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (createdTimeEnd == null || createdTimeEnd <= 0) {
createdTimeEnd = Long.MAX_VALUE;
}
- if (modifiedTimeBegin == null || modifiedTimeBegin <= 0) {
- modifiedTimeBegin = 0L;
- }
- if (modifiedTimeEnd == null || modifiedTimeEnd <= 0) {
- modifiedTimeEnd = Long.MAX_VALUE;
- }
// First sort the selected entities based on created/start time.
Map<Long, Set<TimelineEntity>> sortedEntities =
@@ -318,10 +307,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
createdTimeEnd)) {
continue;
}
- if (!isTimeInRange(entity.getModifiedTime(), modifiedTimeBegin,
- modifiedTimeEnd)) {
- continue;
- }
if (relatesTo != null && !relatesTo.isEmpty() &&
!TimelineStorageUtils
.matchRelations(entity.getRelatesToEntities(), relatesTo)) {
@@ -413,7 +398,6 @@ public class FileSystemTimelineReaderImpl extends AbstractService
public Set<TimelineEntity> getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
@@ -424,8 +408,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
File dir =
new File(new File(rootPath, ENTITIES_DIR),
clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
- return getEntities(dir, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ return getEntities(dir, entityType, limit, createdTimeBegin, createdTimeEnd,
relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index bc48cbe..0ce9a94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -81,7 +81,6 @@ public class HBaseTimelineReaderImpl
public Set<TimelineEntity> getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
@@ -90,10 +89,9 @@ public class HBaseTimelineReaderImpl
TimelineEntityReader reader =
TimelineEntityReaderFactory.createMultipleEntitiesReader(userId,
clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin,
- modifiedTimeEnd, relatesTo, isRelatedTo, infoFilters, configFilters,
- metricFilters, eventFilters, confsToRetrieve, metricsToRetrieve,
- fieldsToRetrieve);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
return reader.readEntities(hbaseConf, conn);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index a57be55..2a82ccb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -315,8 +315,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
ApplicationColumn.ID.store(rowKey, applicationTable, null, te.getId());
ApplicationColumn.CREATED_TIME.store(rowKey, applicationTable, null,
te.getCreatedTime());
- ApplicationColumn.MODIFIED_TIME.store(rowKey, applicationTable, null,
- te.getModifiedTime());
ApplicationColumn.FLOW_VERSION.store(rowKey, applicationTable, null,
flowVersion);
Map<String, Object> info = te.getInfo();
@@ -331,8 +329,6 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
EntityColumn.TYPE.store(rowKey, entityTable, null, te.getType());
EntityColumn.CREATED_TIME.store(rowKey, entityTable, null,
te.getCreatedTime());
- EntityColumn.MODIFIED_TIME.store(rowKey, entityTable, null,
- te.getModifiedTime());
EntityColumn.FLOW_VERSION.store(rowKey, entityTable, null, flowVersion);
Map<String, Object> info = te.getInfo();
if (info != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
index 4c1352c..b5834c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
@@ -143,10 +143,10 @@ public class PhoenixOfflineAggregationWriterImpl
TimelineWriteResponse response = new TimelineWriteResponse();
String sql = "UPSERT INTO " + info.getTableName()
+ " (" + StringUtils.join(info.getPrimaryKeyList(), ",")
- + ", created_time, modified_time, metric_names) "
+ + ", created_time, metric_names) "
+ "VALUES ("
+ StringUtils.repeat("?,", info.getPrimaryKeyList().length)
- + "?, ?, ?)";
+ + "?, ?)";
if (LOG.isDebugEnabled()) {
LOG.debug("TimelineEntity write SQL: " + sql);
}
@@ -162,7 +162,6 @@ public class PhoenixOfflineAggregationWriterImpl
}
int idx = info.setStringsForPrimaryKey(ps, context, null, 1);
ps.setLong(idx++, entity.getCreatedTime());
- ps.setLong(idx++, entity.getModifiedTime());
ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(),
AGGREGATION_STORAGE_SEPARATOR));
ps.execute();
@@ -197,7 +196,7 @@ public class PhoenixOfflineAggregationWriterImpl
+ OfflineAggregationInfo.FLOW_AGGREGATION_TABLE_NAME
+ "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ "flow_name VARCHAR NOT NULL, "
- + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + "created_time UNSIGNED_LONG, "
+ METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ "metric_names VARCHAR, info_keys VARCHAR "
+ "CONSTRAINT pk PRIMARY KEY("
@@ -206,7 +205,7 @@ public class PhoenixOfflineAggregationWriterImpl
sql = "CREATE TABLE IF NOT EXISTS "
+ OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME
+ "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
- + "created_time UNSIGNED_LONG, modified_time UNSIGNED_LONG, "
+ + "created_time UNSIGNED_LONG, "
+ METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ "metric_names VARCHAR, info_keys VARCHAR "
+ "CONSTRAINT pk PRIMARY KEY(user, cluster))";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index 56bd3a0..a5a271e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -87,8 +87,8 @@ public interface TimelineReader extends Service {
* @param fieldsToRetrieve
* Specifies which fields of the entity object to retrieve(optional), see
* {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type, entity created time and entity modified time. All
- * entities will be returned if {@link Field#ALL} is specified.
+ * entity type and entity created time. All entities will be returned if
+ * {@link Field#ALL} is specified.
* @return a {@link TimelineEntity} instance or null. The entity will
* contain the metadata plus the given fields to retrieve.
* @throws IOException
@@ -101,13 +101,13 @@ public interface TimelineReader extends Service {
/**
* <p>The API to search for a set of entities of the given the entity type in
* the scope of the given context which matches the given predicates. The
- * predicates include the created/modified time window, limit to number of
- * entities to be returned, and the entities can be filtered by checking
- * whether they contain the given info/configs entries in the form of
- * key/value pairs, given metrics in the form of metricsIds and its relation
- * with metric values given events in the form of the Ids, and whether they
- * relate to/are related to other entities. For those parameters which have
- * multiple entries, the qualified entity needs to meet all or them.</p>
+ * predicates include the created time window, limit to number of entities to
+ * be returned, and the entities can be filtered by checking whether they
+ * contain the given info/configs entries in the form of key/value pairs,
+ * given metrics in the form of metricsIds and its relation with metric
+ * values, given events in the form of the Ids, and whether they relate to/are
+ * related to other entities. For those parameters which have multiple
+ * entries, the qualified entity needs to meet all or them.</p>
*
* @param userId
* Context user Id(optional).
@@ -130,12 +130,6 @@ public interface TimelineReader extends Service {
* @param createdTimeEnd
* Matched entities should not be created after this timestamp (optional).
* If null or <=0, defaults to {@link Long#MAX_VALUE}.
- * @param modifiedTimeBegin
- * Matched entities should not be modified before this timestamp
- * (optional). If null or <=0, defaults to 0.
- * @param modifiedTimeEnd
- * Matched entities should not be modified after this timestamp (optional).
- * If null or <=0, defaults to {@link Long#MAX_VALUE}.
* @param relatesTo
* Matched entities should relate to given entities (optional).
* @param isRelatedTo
@@ -173,19 +167,17 @@ public interface TimelineReader extends Service {
* @param fieldsToRetrieve
* Specifies which fields of the entity object to retrieve(optional), see
* {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type, entity created time and entity modified time. All
- * entities will be returned if {@link Field#ALL} is specified.
+ * entity type and entity created time. All entities will be returned if
+ * {@link Field#ALL} is specified.
* @return A set of {@link TimelineEntity} instances of the given entity type
* in the given context scope which matches the given predicates
* ordered by created time, descending. Each entity will only contain the
- * metadata(id, type, created and modified times) plus the given fields to
- * retrieve.
+ * metadata(id, type and created time) plus the given fields to retrieve.
* @throws IOException
*/
Set<TimelineEntity> getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
index 802626d..c03c9b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -44,11 +44,6 @@ public enum ApplicationColumn implements Column<ApplicationTable> {
CREATED_TIME(ApplicationColumnFamily.INFO, "created_time"),
/**
- * When it was modified.
- */
- MODIFIED_TIME(ApplicationColumnFamily.INFO, "modified_time"),
-
- /**
* The version of the flow that this app belongs to.
*/
FLOW_VERSION(ApplicationColumnFamily.INFO, "flow_version");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index ee5ff83..681c200 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -50,15 +50,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | flowName! | created_time: | @timestamp1 | |
* | flowRunId! | 1392993084018 | | configKey2: |
* | AppId | | metriciD1: | configValue2 |
- * | | modified_time: | metricValue2 | |
- * | | 1392995081012 | @timestamp2 | |
+ * | | i!infoKey: | metricValue2 | |
+ * | | infoValue | @timestamp2 | |
* | | | | |
- * | | i!infoKey: | metricId2: | |
- * | | infoValue | metricValue1 | |
+ * | | r!relatesToKey: | metricId2: | |
+ * | | id3=id4=id5 | metricValue1 | |
* | | | @timestamp2 | |
- * | | r!relatesToKey: | | |
- * | | id3=id4=id5 | | |
- * | | | | |
* | | s!isRelatedToKey: | | |
* | | id7=id9=id6 | | |
* | | | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index e12b6e0..deb8bd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -49,11 +49,6 @@ public enum EntityColumn implements Column<EntityTable> {
CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
/**
- * When it was modified.
- */
- MODIFIED_TIME(EntityColumnFamily.INFO, "modified_time"),
-
- /**
* The version of the flow that this entity belongs to.
*/
FLOW_VERSION(EntityColumnFamily.INFO, "flow_version");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 15d4465..d7d770b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -48,16 +48,13 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* | clusterId! | | metricValue1 | configValue1 |
* | flowName! | type:entityType | @timestamp1 | |
* | flowRunId! | | | configKey2: |
- * | AppId! | created_time: | metriciD1: | configValue2 |
+ * | AppId! | created_time: | metricId1: | configValue2 |
* | entityType!| 1392993084018 | metricValue2 | |
* | entityId | | @timestamp2 | |
- * | | modified_time: | | |
- * | | 1392995081012 | metricId2: | |
+ * | | i!infoKey: | | |
+ * | | infoValue | metricId1: | |
* | | | metricValue1 | |
- * | | i!infoKey: | @timestamp2 | |
- * | | infoValue | | |
- * | | | | |
- * | | r!relatesToKey: | | |
+ * | | r!relatesToKey: | @timestamp2 | |
* | | id3=id4=id5 | | |
* | | | | |
* | | s!isRelatedToKey | | |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index 181ec81..4d61076 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -63,17 +63,15 @@ class ApplicationEntityReader extends GenericEntityReader {
public ApplicationEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
- true);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, confsToRetrieve,
+ metricsToRetrieve, fieldsToRetrieve, true);
}
public ApplicationEntityReader(String userId, String clusterId,
@@ -230,12 +228,6 @@ class ApplicationEntityReader extends GenericEntityReader {
if (createdTimeEnd == null) {
createdTimeEnd = DEFAULT_END_TIME;
}
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
}
}
@@ -278,15 +270,6 @@ class ApplicationEntityReader extends GenericEntityReader {
return null;
}
- // fetch modified time
- Number modifiedTime =
- (Number)ApplicationColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
// fetch is related to entities
boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
index 52ceef8..048f608 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -52,15 +52,14 @@ class FlowActivityEntityReader extends TimelineEntityReader {
public FlowActivityEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, null, null, fieldsToRetrieve, true);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, null, null,
+ fieldsToRetrieve, true);
}
public FlowActivityEntityReader(String userId, String clusterId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
index 6286ee1..4f50b02 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -61,16 +61,15 @@ class FlowRunEntityReader extends TimelineEntityReader {
public FlowRunEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, null, metricsToRetrieve, fieldsToRetrieve, true);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, null, metricsToRetrieve,
+ fieldsToRetrieve, true);
}
public FlowRunEntityReader(String userId, String clusterId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index f3f380c..990856f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -79,17 +79,15 @@ class GenericEntityReader extends TimelineEntityReader {
public GenericEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
- sortedKeys);
+ createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
+ configFilters, metricFilters, eventFilters, confsToRetrieve,
+ metricsToRetrieve, fieldsToRetrieve, sortedKeys);
}
public GenericEntityReader(String userId, String clusterId,
@@ -257,12 +255,6 @@ class GenericEntityReader extends TimelineEntityReader {
if (createdTimeEnd == null) {
createdTimeEnd = DEFAULT_END_TIME;
}
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
}
}
@@ -314,14 +306,6 @@ class GenericEntityReader extends TimelineEntityReader {
return null;
}
- // fetch modified time
- Number modifiedTime = (Number)EntityColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
// fetch is related to entities
boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
index e801466..bc86b6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -64,8 +64,6 @@ public abstract class TimelineEntityReader {
protected Long limit;
protected Long createdTimeBegin;
protected Long createdTimeEnd;
- protected Long modifiedTimeBegin;
- protected Long modifiedTimeEnd;
protected Map<String, Set<String>> relatesTo;
protected Map<String, Set<String>> isRelatedTo;
protected Map<String, Object> infoFilters;
@@ -94,7 +92,6 @@ public abstract class TimelineEntityReader {
protected TimelineEntityReader(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
@@ -112,8 +109,6 @@ public abstract class TimelineEntityReader {
this.limit = limit;
this.createdTimeBegin = createdTimeBegin;
this.createdTimeEnd = createdTimeEnd;
- this.modifiedTimeBegin = modifiedTimeBegin;
- this.modifiedTimeEnd = modifiedTimeEnd;
this.relatesTo = relatesTo;
this.isRelatedTo = isRelatedTo;
this.infoFilters = infoFilters;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
index c77897a..2e2c652 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -62,7 +62,6 @@ public class TimelineEntityReaderFactory {
public static TimelineEntityReader createMultipleEntitiesReader(String userId,
String clusterId, String flowName, Long flowRunId, String appId,
String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
Map<String, Object> infoFilters, Map<String, String> configFilters,
Set<String> metricFilters, Set<String> eventFilters,
@@ -72,29 +71,25 @@ public class TimelineEntityReaderFactory {
// table are application, flow run, and flow activity entities
if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve);
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ confs, metrics, fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
fieldsToRetrieve);
} else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve);
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ confs, metrics, fieldsToRetrieve);
} else {
// assume we're dealing with a generic entity read
return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve, false);
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
+ isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
+ confs, metrics, fieldsToRetrieve, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 9c74e2d..1aefdca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -174,9 +174,8 @@ public class TestTimelineReaderWebServices {
assertEquals("id_1", entity.getId());
assertEquals("app", entity.getType());
assertEquals(1425016502000L, entity.getCreatedTime());
- assertEquals(1425016503000L, entity.getModifiedTime());
// Default view i.e. when no fields are specified, entity contains only
- // entity id, entity type, created and modified time.
+ // entity id, entity type and created time.
assertEquals(0, entity.getConfigs().size());
assertEquals(0, entity.getMetrics().size());
} finally {
@@ -198,7 +197,6 @@ public class TestTimelineReaderWebServices {
assertEquals("id_1", entity.getId());
assertEquals("app", entity.getType());
assertEquals(1425016502000L, entity.getCreatedTime());
- assertEquals(1425016503000L, entity.getModifiedTime());
} finally {
client.destroy();
}
@@ -382,50 +380,6 @@ public class TestTimelineReaderWebServices {
}
@Test
- public void testGetEntitiesBasedOnModifiedTime() throws Exception {
- Client client = createClient();
- try {
- URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?modifiedtimestart=1425016502090"
- + "&modifiedtimeend=1425016503020");
- ClientResponse resp = getResponse(client, uri);
- Set<TimelineEntity> entities =
- resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(2, entities.size());
- assertTrue("Entities with id_1 and id_4 should have been" +
- " present in response.",
- entities.contains(newEntity("app", "id_1")) &&
- entities.contains(newEntity("app", "id_4")));
-
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?modifiedtimeend=1425016502090");
- resp = getResponse(client, uri);
- entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(2, entities.size());
- assertTrue("Entities with id_2 and id_3 should have been " +
- "present in response.",
- entities.contains(newEntity("app", "id_2")) &&
- entities.contains(newEntity("app", "id_3")));
-
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?modifiedtimestart=1425016503005");
- resp = getResponse(client, uri);
- entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
- assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
- assertNotNull(entities);
- assertEquals(1, entities.size());
- assertTrue("Entity with id_4 should have been present in response.",
- entities.contains(newEntity("app", "id_4")));
- } finally {
- client.destroy();
- }
- }
-
- @Test
public void testGetEntitiesByRelations() throws Exception {
Client client = createClient();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: MAPREDUCE-6546. reconcile the two
versions of the timeline service performance tests. (Sangjin Lee via
Naganarasimha G R)
Posted by gt...@apache.org.
MAPREDUCE-6546. reconcile the two versions of the timeline service performance tests. (Sangjin Lee via Naganarasimha G R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cd444089
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cd444089
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cd444089
Branch: refs/heads/YARN-2928
Commit: cd4440899a2cebbbfb15e18228d3821961319986
Parents: 34248bd
Author: naganarasimha <na...@apache.com>
Authored: Wed Mar 9 11:20:32 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:05 2016 -0700
----------------------------------------------------------------------
.../hadoop/mapred/JobHistoryFileParser.java | 53 ----
.../mapred/JobHistoryFileReplayMapper.java | 301 -------------------
.../hadoop/mapred/SimpleEntityWriter.java | 140 ---------
.../hadoop/mapred/TimelineEntityConverter.java | 211 -------------
.../mapred/TimelineServicePerformanceV2.java | 229 --------------
.../apache/hadoop/mapreduce/EntityWriterV2.java | 56 ++++
.../mapreduce/JobHistoryFileReplayMapperV1.java | 14 +-
.../mapreduce/JobHistoryFileReplayMapperV2.java | 161 ++++++++++
.../mapreduce/SimpleEntityWriterConstants.java | 43 +++
.../hadoop/mapreduce/SimpleEntityWriterV1.java | 28 +-
.../hadoop/mapreduce/SimpleEntityWriterV2.java | 131 ++++++++
.../mapreduce/TimelineEntityConverterV1.java | 5 -
.../mapreduce/TimelineEntityConverterV2.java | 211 +++++++++++++
.../mapreduce/TimelineServicePerformance.java | 129 +++++---
.../apache/hadoop/test/MapredTestDriver.java | 35 +--
15 files changed, 704 insertions(+), 1043 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
deleted file mode 100644
index 9d051df..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileParser.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
-
-class JobHistoryFileParser {
- private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
-
- private final FileSystem fs;
-
- public JobHistoryFileParser(FileSystem fs) {
- LOG.info("JobHistoryFileParser created with " + fs);
- this.fs = fs;
- }
-
- public JobInfo parseHistoryFile(Path path) throws IOException {
- LOG.info("parsing job history file " + path);
- JobHistoryParser parser = new JobHistoryParser(fs, path);
- return parser.parse();
- }
-
- public Configuration parseConfiguration(Path path) throws IOException {
- LOG.info("parsing job configuration file " + path);
- Configuration conf = new Configuration(false);
- conf.addResource(fs.open(path));
- return conf;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
deleted file mode 100644
index 4fb5308..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/JobHistoryFileReplayMapper.java
+++ /dev/null
@@ -1,301 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
-import org.apache.hadoop.mapred.TimelineServicePerformanceV2.EntityWriter;
-import org.apache.hadoop.mapred.TimelineServicePerformanceV2.PerfCounters;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.TypeConverter;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
-import org.apache.hadoop.mapreduce.v2.api.records.JobId;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
-
-/**
- * Mapper for TimelineServicePerformanceV2 that replays job history files to the
- * timeline service.
- *
- */
-class JobHistoryFileReplayMapper extends EntityWriter {
- private static final Log LOG =
- LogFactory.getLog(JobHistoryFileReplayMapper.class);
-
- static final String PROCESSING_PATH = "processing path";
- static final String REPLAY_MODE = "replay mode";
- static final int WRITE_ALL_AT_ONCE = 1;
- static final int WRITE_PER_ENTITY = 2;
- static final int REPLAY_MODE_DEFAULT = WRITE_ALL_AT_ONCE;
-
- private static final Pattern JOB_ID_PARSER =
- Pattern.compile("^(job_[0-9]+_([0-9]+)).*");
-
- public static class JobFiles {
- private final String jobId;
- private Path jobHistoryFilePath;
- private Path jobConfFilePath;
-
- public JobFiles(String jobId) {
- this.jobId = jobId;
- }
-
- public String getJobId() {
- return jobId;
- }
-
- public Path getJobHistoryFilePath() {
- return jobHistoryFilePath;
- }
-
- public void setJobHistoryFilePath(Path jobHistoryFilePath) {
- this.jobHistoryFilePath = jobHistoryFilePath;
- }
-
- public Path getJobConfFilePath() {
- return jobConfFilePath;
- }
-
- public void setJobConfFilePath(Path jobConfFilePath) {
- this.jobConfFilePath = jobConfFilePath;
- }
-
- @Override
- public int hashCode() {
- return jobId.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (getClass() != obj.getClass()) {
- return false;
- }
- JobFiles other = (JobFiles) obj;
- return jobId.equals(other.jobId);
- }
- }
-
- private enum FileType { JOB_HISTORY_FILE, JOB_CONF_FILE, UNKNOWN }
-
-
- @Override
- protected void writeEntities(Configuration tlConf,
- TimelineCollectorManager manager, Context context) throws IOException {
- // collect the apps it needs to process
- Configuration conf = context.getConfiguration();
- int taskId = context.getTaskAttemptID().getTaskID().getId();
- int size = conf.getInt(MRJobConfig.NUM_MAPS,
- TimelineServicePerformanceV2.NUM_MAPS_DEFAULT);
- String processingDir =
- conf.get(JobHistoryFileReplayMapper.PROCESSING_PATH);
- int replayMode =
- conf.getInt(JobHistoryFileReplayMapper.REPLAY_MODE,
- JobHistoryFileReplayMapper.REPLAY_MODE_DEFAULT);
- Path processingPath = new Path(processingDir);
- FileSystem processingFs = processingPath.getFileSystem(conf);
- JobHistoryFileParser parser = new JobHistoryFileParser(processingFs);
- TimelineEntityConverter converter = new TimelineEntityConverter();
-
- Collection<JobFiles> jobs =
- selectJobFiles(processingFs, processingPath, taskId, size);
- if (jobs.isEmpty()) {
- LOG.info(context.getTaskAttemptID().getTaskID() +
- " will process no jobs");
- } else {
- LOG.info(context.getTaskAttemptID().getTaskID() + " will process " +
- jobs.size() + " jobs");
- }
- for (JobFiles job: jobs) {
- // process each job
- String jobIdStr = job.getJobId();
- LOG.info("processing " + jobIdStr + "...");
- JobId jobId = TypeConverter.toYarn(JobID.forName(jobIdStr));
- ApplicationId appId = jobId.getAppId();
-
- // create the app level timeline collector and start it
- AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId);
- manager.putIfAbsent(appId, collector);
- try {
- // parse the job info and configuration
- JobInfo jobInfo =
- parser.parseHistoryFile(job.getJobHistoryFilePath());
- Configuration jobConf =
- parser.parseConfiguration(job.getJobConfFilePath());
- LOG.info("parsed the job history file and the configuration file for job"
- + jobIdStr);
-
- // set the context
- // flow id: job name, flow run id: timestamp, user id
- TimelineCollectorContext tlContext =
- collector.getTimelineEntityContext();
- tlContext.setFlowName(jobInfo.getJobname());
- tlContext.setFlowRunId(jobInfo.getSubmitTime());
- tlContext.setUserId(jobInfo.getUsername());
-
- // create entities from job history and write them
- long totalTime = 0;
- List<TimelineEntity> entitySet =
- converter.createTimelineEntities(jobInfo, jobConf);
- LOG.info("converted them into timeline entities for job " + jobIdStr);
- // use the current user for this purpose
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- long startWrite = System.nanoTime();
- try {
- switch (replayMode) {
- case JobHistoryFileReplayMapper.WRITE_ALL_AT_ONCE:
- writeAllEntities(collector, entitySet, ugi);
- break;
- case JobHistoryFileReplayMapper.WRITE_PER_ENTITY:
- writePerEntity(collector, entitySet, ugi);
- break;
- default:
- break;
- }
- } catch (Exception e) {
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
- increment(1);
- LOG.error("writing to the timeline service failed", e);
- }
- long endWrite = System.nanoTime();
- totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
- int numEntities = entitySet.size();
- LOG.info("wrote " + numEntities + " entities in " + totalTime + " ms");
-
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
- increment(totalTime);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
- increment(numEntities);
- } finally {
- manager.remove(appId);
- context.progress(); // move it along
- }
- }
- }
-
- private void writeAllEntities(AppLevelTimelineCollector collector,
- List<TimelineEntity> entitySet, UserGroupInformation ugi)
- throws IOException {
- TimelineEntities entities = new TimelineEntities();
- entities.setEntities(entitySet);
- collector.putEntities(entities, ugi);
- }
-
- private void writePerEntity(AppLevelTimelineCollector collector,
- List<TimelineEntity> entitySet, UserGroupInformation ugi)
- throws IOException {
- for (TimelineEntity entity : entitySet) {
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
- collector.putEntities(entities, ugi);
- LOG.info("wrote entity " + entity.getId());
- }
- }
-
- private Collection<JobFiles> selectJobFiles(FileSystem fs,
- Path processingRoot, int i, int size) throws IOException {
- Map<String,JobFiles> jobs = new HashMap<>();
- RemoteIterator<LocatedFileStatus> it = fs.listFiles(processingRoot, true);
- while (it.hasNext()) {
- LocatedFileStatus status = it.next();
- Path path = status.getPath();
- String fileName = path.getName();
- Matcher m = JOB_ID_PARSER.matcher(fileName);
- if (!m.matches()) {
- continue;
- }
- String jobId = m.group(1);
- int lastId = Integer.parseInt(m.group(2));
- int mod = lastId % size;
- if (mod != i) {
- continue;
- }
- LOG.info("this mapper will process file " + fileName);
- // it's mine
- JobFiles jobFiles = jobs.get(jobId);
- if (jobFiles == null) {
- jobFiles = new JobFiles(jobId);
- jobs.put(jobId, jobFiles);
- }
- setFilePath(fileName, path, jobFiles);
- }
- return jobs.values();
- }
-
- private void setFilePath(String fileName, Path path,
- JobFiles jobFiles) {
- // determine if we're dealing with a job history file or a job conf file
- FileType type = getFileType(fileName);
- switch (type) {
- case JOB_HISTORY_FILE:
- if (jobFiles.getJobHistoryFilePath() == null) {
- jobFiles.setJobHistoryFilePath(path);
- } else {
- LOG.warn("we already have the job history file " +
- jobFiles.getJobHistoryFilePath() + ": skipping " + path);
- }
- break;
- case JOB_CONF_FILE:
- if (jobFiles.getJobConfFilePath() == null) {
- jobFiles.setJobConfFilePath(path);
- } else {
- LOG.warn("we already have the job conf file " +
- jobFiles.getJobConfFilePath() + ": skipping " + path);
- }
- break;
- case UNKNOWN:
- LOG.warn("unknown type: " + path);
- }
- }
-
- private FileType getFileType(String fileName) {
- if (fileName.endsWith(".jhist")) {
- return FileType.JOB_HISTORY_FILE;
- }
- if (fileName.endsWith("_conf.xml")) {
- return FileType.JOB_CONF_FILE;
- }
- return FileType.UNKNOWN;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
deleted file mode 100644
index 625c32a..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SimpleEntityWriter.java
+++ /dev/null
@@ -1,140 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Random;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapred.TimelineServicePerformanceV2.EntityWriter;
-import org.apache.hadoop.mapred.TimelineServicePerformanceV2.PerfCounters;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
-
-/**
- * Adds simple entities with random string payload, events, metrics, and
- * configuration.
- */
-class SimpleEntityWriter extends EntityWriter {
- private static final Log LOG = LogFactory.getLog(SimpleEntityWriter.class);
-
- // constants for mtype = 1
- static final String KBS_SENT = "kbs sent";
- static final int KBS_SENT_DEFAULT = 1;
- static final String TEST_TIMES = "testtimes";
- static final int TEST_TIMES_DEFAULT = 100;
- static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
- "timeline.server.performance.run.id";
-
- protected void writeEntities(Configuration tlConf,
- TimelineCollectorManager manager, Context context) throws IOException {
- Configuration conf = context.getConfiguration();
- // simulate the app id with the task id
- int taskId = context.getTaskAttemptID().getTaskID().getId();
- long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
- ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
-
- // create the app level timeline collector
- AppLevelTimelineCollector collector =
- new AppLevelTimelineCollector(appId);
- manager.putIfAbsent(appId, collector);
-
- try {
- // set the context
- // flow id: job name, flow run id: timestamp, user id
- TimelineCollectorContext tlContext =
- collector.getTimelineEntityContext();
- tlContext.setFlowName(context.getJobName());
- tlContext.setFlowRunId(timestamp);
- tlContext.setUserId(context.getUser());
-
- final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
-
- long totalTime = 0;
- final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
- final Random rand = new Random();
- final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
- final char[] payLoad = new char[kbs * 1024];
-
- for (int i = 0; i < testtimes; i++) {
- // Generate a fixed length random payload
- for (int xx = 0; xx < kbs * 1024; xx++) {
- int alphaNumIdx =
- rand.nextInt(TimelineServicePerformanceV2.alphaNums.length);
- payLoad[xx] = TimelineServicePerformanceV2.alphaNums[alphaNumIdx];
- }
- String entId = taskAttemptId + "_" + Integer.toString(i);
- final TimelineEntity entity = new TimelineEntity();
- entity.setId(entId);
- entity.setType("FOO_ATTEMPT");
- entity.addInfo("PERF_TEST", payLoad);
- // add an event
- TimelineEvent event = new TimelineEvent();
- event.setId("foo_event_id");
- event.setTimestamp(System.currentTimeMillis());
- event.addInfo("foo_event", "test");
- entity.addEvent(event);
- // add a metric
- TimelineMetric metric = new TimelineMetric();
- metric.setId("foo_metric");
- metric.addValue(System.currentTimeMillis(), 123456789L);
- entity.addMetric(metric);
- // add a config
- entity.addConfig("foo", "bar");
-
- TimelineEntities entities = new TimelineEntities();
- entities.addEntity(entity);
- // use the current user for this purpose
- UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
- long startWrite = System.nanoTime();
- try {
- collector.putEntities(entities, ugi);
- } catch (Exception e) {
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
- increment(1);
- LOG.error("writing to the timeline service failed", e);
- }
- long endWrite = System.nanoTime();
- totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
- }
- LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
- " kB) in " + totalTime + " ms");
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
- increment(totalTime);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
- increment(testtimes);
- context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
- increment(kbs*testtimes);
- } finally {
- // clean up
- manager.remove(appId);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
deleted file mode 100644
index 0e2eb72..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
+++ /dev/null
@@ -1,211 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskID;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
-import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-
-class TimelineEntityConverter {
- private static final Log LOG =
- LogFactory.getLog(TimelineEntityConverter.class);
-
- static final String JOB = "MAPREDUCE_JOB";
- static final String TASK = "MAPREDUCE_TASK";
- static final String TASK_ATTEMPT = "MAPREDUCE_TASK_ATTEMPT";
-
- /**
- * Creates job, task, and task attempt entities based on the job history info
- * and configuration.
- *
- * Note: currently these are plan timeline entities created for mapreduce
- * types. These are not meant to be the complete and accurate entity set-up
- * for mapreduce jobs. We do not leverage hierarchical timeline entities. If
- * we create canonical mapreduce hierarchical timeline entities with proper
- * parent-child relationship, we could modify this to use that instead.
- *
- * Note that we also do not add info to the YARN application entity, which
- * would be needed for aggregation.
- */
- public List<TimelineEntity> createTimelineEntities(JobInfo jobInfo,
- Configuration conf) {
- List<TimelineEntity> entities = new ArrayList<>();
-
- // create the job entity
- TimelineEntity job = createJobEntity(jobInfo, conf);
- entities.add(job);
-
- // create the task and task attempt entities
- List<TimelineEntity> tasksAndAttempts =
- createTaskAndTaskAttemptEntities(jobInfo);
- entities.addAll(tasksAndAttempts);
-
- return entities;
- }
-
- private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
- TimelineEntity job = new TimelineEntity();
- job.setType(JOB);
- job.setId(jobInfo.getJobId().toString());
- job.setCreatedTime(jobInfo.getSubmitTime());
-
- job.addInfo("JOBNAME", jobInfo.getJobname());
- job.addInfo("USERNAME", jobInfo.getUsername());
- job.addInfo("JOB_QUEUE_NAME", jobInfo.getJobQueueName());
- job.addInfo("SUBMIT_TIME", jobInfo.getSubmitTime());
- job.addInfo("LAUNCH_TIME", jobInfo.getLaunchTime());
- job.addInfo("FINISH_TIME", jobInfo.getFinishTime());
- job.addInfo("JOB_STATUS", jobInfo.getJobStatus());
- job.addInfo("PRIORITY", jobInfo.getPriority());
- job.addInfo("TOTAL_MAPS", jobInfo.getTotalMaps());
- job.addInfo("TOTAL_REDUCES", jobInfo.getTotalReduces());
- job.addInfo("UBERIZED", jobInfo.getUberized());
- job.addInfo("ERROR_INFO", jobInfo.getErrorInfo());
-
- // add metrics from total counters
- // we omit the map counters and reduce counters for now as it's kind of
- // awkward to put them (map/reduce/total counters are really a group of
- // related counters)
- Counters totalCounters = jobInfo.getTotalCounters();
- if (totalCounters != null) {
- addMetrics(job, totalCounters);
- }
- // finally add configuration to the job
- addConfiguration(job, conf);
- LOG.info("converted job " + jobInfo.getJobId() + " to a timeline entity");
- return job;
- }
-
- private void addConfiguration(TimelineEntity job, Configuration conf) {
- for (Map.Entry<String,String> e: conf) {
- job.addConfig(e.getKey(), e.getValue());
- }
- }
-
- private void addMetrics(TimelineEntity entity, Counters counters) {
- for (CounterGroup g: counters) {
- String groupName = g.getName();
- for (Counter c: g) {
- String name = groupName + ":" + c.getName();
- TimelineMetric metric = new TimelineMetric();
- metric.setId(name);
- metric.addValue(System.currentTimeMillis(), c.getValue());
- entity.addMetric(metric);
- }
- }
- }
-
- private List<TimelineEntity> createTaskAndTaskAttemptEntities(
- JobInfo jobInfo) {
- List<TimelineEntity> entities = new ArrayList<>();
- Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
- LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
- " tasks");
- for (TaskInfo taskInfo: taskInfoMap.values()) {
- TimelineEntity task = createTaskEntity(taskInfo);
- entities.add(task);
- // add the task attempts from this task
- Set<TimelineEntity> taskAttempts = createTaskAttemptEntities(taskInfo);
- entities.addAll(taskAttempts);
- }
- return entities;
- }
-
- private TimelineEntity createTaskEntity(TaskInfo taskInfo) {
- TimelineEntity task = new TimelineEntity();
- task.setType(TASK);
- task.setId(taskInfo.getTaskId().toString());
- task.setCreatedTime(taskInfo.getStartTime());
-
- task.addInfo("START_TIME", taskInfo.getStartTime());
- task.addInfo("FINISH_TIME", taskInfo.getFinishTime());
- task.addInfo("TASK_TYPE", taskInfo.getTaskType());
- task.addInfo("TASK_STATUS", taskInfo.getTaskStatus());
- task.addInfo("ERROR_INFO", taskInfo.getError());
-
- // add metrics from counters
- Counters counters = taskInfo.getCounters();
- if (counters != null) {
- addMetrics(task, counters);
- }
- LOG.info("converted task " + taskInfo.getTaskId() +
- " to a timeline entity");
- return task;
- }
-
- private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
- Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
- Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap =
- taskInfo.getAllTaskAttempts();
- LOG.info("task " + taskInfo.getTaskId() + " has " +
- taskAttemptInfoMap.size() + " task attempts");
- for (TaskAttemptInfo taskAttemptInfo: taskAttemptInfoMap.values()) {
- TimelineEntity taskAttempt = createTaskAttemptEntity(taskAttemptInfo);
- taskAttempts.add(taskAttempt);
- }
- return taskAttempts;
- }
-
- private TimelineEntity createTaskAttemptEntity(
- TaskAttemptInfo taskAttemptInfo) {
- TimelineEntity taskAttempt = new TimelineEntity();
- taskAttempt.setType(TASK_ATTEMPT);
- taskAttempt.setId(taskAttemptInfo.getAttemptId().toString());
- taskAttempt.setCreatedTime(taskAttemptInfo.getStartTime());
-
- taskAttempt.addInfo("START_TIME", taskAttemptInfo.getStartTime());
- taskAttempt.addInfo("FINISH_TIME", taskAttemptInfo.getFinishTime());
- taskAttempt.addInfo("MAP_FINISH_TIME",
- taskAttemptInfo.getMapFinishTime());
- taskAttempt.addInfo("SHUFFLE_FINISH_TIME",
- taskAttemptInfo.getShuffleFinishTime());
- taskAttempt.addInfo("SORT_FINISH_TIME",
- taskAttemptInfo.getSortFinishTime());
- taskAttempt.addInfo("TASK_STATUS", taskAttemptInfo.getTaskStatus());
- taskAttempt.addInfo("STATE", taskAttemptInfo.getState());
- taskAttempt.addInfo("ERROR", taskAttemptInfo.getError());
- taskAttempt.addInfo("CONTAINER_ID",
- taskAttemptInfo.getContainerId().toString());
-
- // add metrics from counters
- Counters counters = taskAttemptInfo.getCounters();
- if (counters != null) {
- addMetrics(taskAttempt, counters);
- }
- LOG.info("converted task attempt " + taskAttemptInfo.getAttemptId() +
- " to a timeline entity");
- return taskAttempt;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
deleted file mode 100644
index f674ae1..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineServicePerformanceV2.java
+++ /dev/null
@@ -1,229 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.mapred;
-
-import java.io.IOException;
-import java.util.Date;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.SleepJob.SleepInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.Tool;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
-
-public class TimelineServicePerformanceV2 extends Configured implements Tool {
- static final int NUM_MAPS_DEFAULT = 1;
-
- static final int SIMPLE_ENTITY_WRITER = 1;
- static final int JOB_HISTORY_FILE_REPLAY_MAPPER = 2;
- static int mapperType = SIMPLE_ENTITY_WRITER;
-
- protected static int printUsage() {
- System.err.println(
- "Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
- ")\n" +
- " [-mtype <mapper type in integer>]\n" +
- " 1. simple entity write mapper\n" +
- " 2. job history file replay mapper\n" +
- " [-s <(KBs)test>] number of KB per put (mtype=1, default: " +
- SimpleEntityWriter.KBS_SENT_DEFAULT + " KB)\n" +
- " [-t] package sending iterations per mapper (mtype=1, default: " +
- SimpleEntityWriter.TEST_TIMES_DEFAULT + ")\n" +
- " [-d <path>] root path of job history files (mtype=2)\n" +
- " [-r <replay mode>] (mtype=2)\n" +
- " 1. write all entities for a job in one put (default)\n" +
- " 2. write one entity at a time\n");
- GenericOptionsParser.printGenericCommandUsage(System.err);
- return -1;
- }
-
- /**
- * Configure a job given argv.
- */
- public static boolean parseArgs(String[] args, Job job) throws IOException {
- // set the common defaults
- Configuration conf = job.getConfiguration();
- conf.setInt(MRJobConfig.NUM_MAPS, NUM_MAPS_DEFAULT);
-
- for (int i = 0; i < args.length; i++) {
- if (args.length == i + 1) {
- System.out.println("ERROR: Required parameter missing from " + args[i]);
- return printUsage() == 0;
- }
- try {
- if ("-m".equals(args[i])) {
- if (Integer.parseInt(args[++i]) > 0) {
- job.getConfiguration()
- .setInt(MRJobConfig.NUM_MAPS, Integer.parseInt(args[i]));
- }
- } else if ("-mtype".equals(args[i])) {
- mapperType = Integer.parseInt(args[++i]);
- } else if ("-s".equals(args[i])) {
- if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(SimpleEntityWriter.KBS_SENT, Integer.parseInt(args[i]));
- }
- } else if ("-t".equals(args[i])) {
- if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(SimpleEntityWriter.TEST_TIMES,
- Integer.parseInt(args[i]));
- }
- } else if ("-d".equals(args[i])) {
- conf.set(JobHistoryFileReplayMapper.PROCESSING_PATH, args[++i]);
- } else if ("-r".equals(args[i])) {
- conf.setInt(JobHistoryFileReplayMapper.REPLAY_MODE,
- Integer.parseInt(args[++i]));
- } else {
- System.out.println("Unexpected argument: " + args[i]);
- return printUsage() == 0;
- }
- } catch (NumberFormatException except) {
- System.out.println("ERROR: Integer expected instead of " + args[i]);
- return printUsage() == 0;
- } catch (Exception e) {
- throw (IOException)new IOException().initCause(e);
- }
- }
-
- // handle mapper-specific settings
- switch (mapperType) {
- case JOB_HISTORY_FILE_REPLAY_MAPPER:
- job.setMapperClass(JobHistoryFileReplayMapper.class);
- String processingPath =
- conf.get(JobHistoryFileReplayMapper.PROCESSING_PATH);
- if (processingPath == null || processingPath.isEmpty()) {
- System.out.println("processing path is missing while mtype = 2");
- return printUsage() == 0;
- }
- break;
- case SIMPLE_ENTITY_WRITER:
- default:
- job.setMapperClass(SimpleEntityWriter.class);
- // use the current timestamp as the "run id" of the test: this will
- // be used as simulating the cluster timestamp for apps
- conf.setLong(SimpleEntityWriter.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
- System.currentTimeMillis());
- break;
- }
-
- return true;
- }
-
- /**
- * TimelineServer Performance counters
- */
- static enum PerfCounters {
- TIMELINE_SERVICE_WRITE_TIME,
- TIMELINE_SERVICE_WRITE_COUNTER,
- TIMELINE_SERVICE_WRITE_FAILURES,
- TIMELINE_SERVICE_WRITE_KBS,
- }
-
- public int run(String[] args) throws Exception {
-
- Job job = Job.getInstance(getConf());
- job.setJarByClass(TimelineServicePerformanceV2.class);
- job.setMapperClass(SimpleEntityWriter.class);
- job.setInputFormatClass(SleepInputFormat.class);
- job.setOutputFormatClass(NullOutputFormat.class);
- job.setNumReduceTasks(0);
- if (!parseArgs(args, job)) {
- return -1;
- }
-
- Date startTime = new Date();
- System.out.println("Job started: " + startTime);
- int ret = job.waitForCompletion(true) ? 0 : 1;
- org.apache.hadoop.mapreduce.Counters counters = job.getCounters();
- long writetime =
- counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).getValue();
- long writecounts =
- counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).getValue();
- long writesize =
- counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
- double transacrate = writecounts * 1000 / (double)writetime;
- double iorate = writesize * 1000 / (double)writetime;
- int numMaps =
- Integer.parseInt(job.getConfiguration().get(MRJobConfig.NUM_MAPS));
-
- System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
- " ops/s");
- System.out.println("IO RATE (per mapper): " + iorate + " KB/s");
-
- System.out.println("TRANSACTION RATE (total): " + transacrate*numMaps +
- " ops/s");
- System.out.println("IO RATE (total): " + iorate*numMaps + " KB/s");
-
- return ret;
- }
-
- public static void main(String[] args) throws Exception {
- int res =
- ToolRunner.run(new Configuration(), new TimelineServicePerformanceV2(),
- args);
- System.exit(res);
- }
-
- /**
- * To ensure that the compression really gets exercised, generate a
- * random alphanumeric fixed length payload
- */
- static final char[] alphaNums = new char[] { 'a', 'b', 'c', 'd', 'e', 'f',
- 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
- 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
- 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
- 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
- '3', '4', '5', '6', '7', '8', '9', '0', ' ' };
-
- /**
- * Base mapper for writing entities to the timeline service. Subclasses
- * override {@link #writeEntities(Configuration, TimelineCollectorManager,
- * org.apache.hadoop.mapreduce.Mapper.Context)} to create and write entities
- * to the timeline service.
- */
- public static abstract class EntityWriter
- extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
- @Override
- public void map(IntWritable key, IntWritable val, Context context)
- throws IOException {
-
- // create the timeline collector manager wired with the writer
- Configuration tlConf = new YarnConfiguration();
- TimelineCollectorManager manager = new TimelineCollectorManager("test");
- manager.init(tlConf);
- manager.start();
- try {
- // invoke the method to have the subclass write entities
- writeEntities(tlConf, manager, context);
- } finally {
- manager.close();
- }
- }
-
- protected abstract void writeEntities(Configuration tlConf,
- TimelineCollectorManager manager, Context context) throws IOException;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/EntityWriterV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/EntityWriterV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/EntityWriterV2.java
new file mode 100644
index 0000000..f5d95c3
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/EntityWriterV2.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Base mapper for writing entities to the timeline service. Subclasses
+ * override {@link #writeEntities(Configuration, TimelineCollectorManager,
+ * org.apache.hadoop.mapreduce.Mapper.Context)} to create and write entities
+ * to the timeline service.
+ */
+abstract class EntityWriterV2
+ extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
+ @Override
+ public void map(IntWritable key, IntWritable val, Context context)
+ throws IOException {
+
+ // create the timeline collector manager wired with the writer
+ Configuration tlConf = new YarnConfiguration();
+ TimelineCollectorManager manager = new TimelineCollectorManager("test");
+ manager.init(tlConf);
+ manager.start();
+ try {
+ // invoke the method to have the subclass write entities
+ writeEntities(tlConf, manager, context);
+ } finally {
+ manager.close();
+ }
+ }
+
+ protected abstract void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
index 5e10662..447ea4e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV1.java
@@ -20,33 +20,21 @@ package org.apache.hadoop.mapreduce;
import java.io.IOException;
import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
-import org.apache.hadoop.mapreduce.TimelineServicePerformance.PerfCounters;
-import org.apache.hadoop.mapreduce.JobHistoryFileReplayHelper;
import org.apache.hadoop.mapreduce.JobHistoryFileReplayHelper.JobFiles;
-import org.apache.hadoop.mapreduce.MRJobConfig;
-import org.apache.hadoop.mapreduce.TypeConverter;
+import org.apache.hadoop.mapreduce.TimelineServicePerformance.PerfCounters;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
new file mode 100644
index 0000000..6a9a878
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileReplayMapperV2.java
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapred.JobID;
+import org.apache.hadoop.mapreduce.JobHistoryFileReplayHelper.JobFiles;
+import org.apache.hadoop.mapreduce.TimelineServicePerformance.PerfCounters;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.v2.api.records.JobId;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Mapper for TimelineServicePerformance that replays job history files to the
+ * timeline service v.2.
+ *
+ */
+class JobHistoryFileReplayMapperV2 extends EntityWriterV2 {
+ private static final Log LOG =
+ LogFactory.getLog(JobHistoryFileReplayMapperV2.class);
+
+ @Override
+ protected void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException {
+ JobHistoryFileReplayHelper helper = new JobHistoryFileReplayHelper(context);
+ int replayMode = helper.getReplayMode();
+ JobHistoryFileParser parser = helper.getParser();
+ TimelineEntityConverterV2 converter = new TimelineEntityConverterV2();
+
+ // collect the apps it needs to process
+ Collection<JobFiles> jobs = helper.getJobFiles();
+ if (jobs.isEmpty()) {
+ LOG.info(context.getTaskAttemptID().getTaskID() +
+ " will process no jobs");
+ } else {
+ LOG.info(context.getTaskAttemptID().getTaskID() + " will process " +
+ jobs.size() + " jobs");
+ }
+ for (JobFiles job: jobs) {
+ // process each job
+ String jobIdStr = job.getJobId();
+ // skip if either of the file is missing
+ if (job.getJobConfFilePath() == null ||
+ job.getJobHistoryFilePath() == null) {
+ LOG.info(jobIdStr + " missing either the job history file or the " +
+ "configuration file. Skipping.");
+ continue;
+ }
+ LOG.info("processing " + jobIdStr + "...");
+ JobId jobId = TypeConverter.toYarn(JobID.forName(jobIdStr));
+ ApplicationId appId = jobId.getAppId();
+
+ // create the app level timeline collector and start it
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ manager.putIfAbsent(appId, collector);
+ try {
+ // parse the job info and configuration
+ JobInfo jobInfo =
+ parser.parseHistoryFile(job.getJobHistoryFilePath());
+ Configuration jobConf =
+ parser.parseConfiguration(job.getJobConfFilePath());
+ LOG.info("parsed the job history file and the configuration file for job"
+ + jobIdStr);
+
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(jobInfo.getJobname());
+ tlContext.setFlowRunId(jobInfo.getSubmitTime());
+ tlContext.setUserId(jobInfo.getUsername());
+
+ // create entities from job history and write them
+ long totalTime = 0;
+ List<TimelineEntity> entitySet =
+ converter.createTimelineEntities(jobInfo, jobConf);
+ LOG.info("converted them into timeline entities for job " + jobIdStr);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ switch (replayMode) {
+ case JobHistoryFileReplayHelper.WRITE_ALL_AT_ONCE:
+ writeAllEntities(collector, entitySet, ugi);
+ break;
+ case JobHistoryFileReplayHelper.WRITE_PER_ENTITY:
+ writePerEntity(collector, entitySet, ugi);
+ break;
+ default:
+ break;
+ }
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ LOG.error("writing to the timeline service failed", e);
+ }
+ long endWrite = System.nanoTime();
+ totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
+ int numEntities = entitySet.size();
+ LOG.info("wrote " + numEntities + " entities in " + totalTime + " ms");
+
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(numEntities);
+ } finally {
+ manager.remove(appId);
+ context.progress(); // move it along
+ }
+ }
+ }
+
+ private void writeAllEntities(AppLevelTimelineCollector collector,
+ List<TimelineEntity> entitySet, UserGroupInformation ugi)
+ throws IOException {
+ TimelineEntities entities = new TimelineEntities();
+ entities.setEntities(entitySet);
+ collector.putEntities(entities, ugi);
+ }
+
+ private void writePerEntity(AppLevelTimelineCollector collector,
+ List<TimelineEntity> entitySet, UserGroupInformation ugi)
+ throws IOException {
+ for (TimelineEntity entity : entitySet) {
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ collector.putEntities(entities, ugi);
+ LOG.info("wrote entity " + entity.getId());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterConstants.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterConstants.java
new file mode 100644
index 0000000..b89d0e8
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterConstants.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+/**
+ * Constants for simple entity writers.
+ */
+interface SimpleEntityWriterConstants {
+ // constants for mtype = 1
+ String KBS_SENT = "kbs sent";
+ int KBS_SENT_DEFAULT = 1;
+ String TEST_TIMES = "testtimes";
+ int TEST_TIMES_DEFAULT = 100;
+ String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
+ "timeline.server.performance.run.id";
+
+ /**
+ * To ensure that the compression really gets exercised, generate a
+ * random alphanumeric fixed length payload
+ */
+ char[] ALPHA_NUMS = new char[] { 'a', 'b', 'c', 'd', 'e', 'f',
+ 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
+ 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
+ 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
+ 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
+ '3', '4', '5', '6', '7', '8', '9', '0', ' ' };
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
index 2c851e9..b10ae04 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV1.java
@@ -27,44 +27,22 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.mapreduce.Mapper.Context;
import org.apache.hadoop.mapreduce.TimelineServicePerformance.PerfCounters;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
/**
* Adds simple entities with random string payload, events, metrics, and
* configuration.
*/
-class SimpleEntityWriterV1 extends
- org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable> {
+class SimpleEntityWriterV1
+ extends org.apache.hadoop.mapreduce.Mapper<IntWritable,IntWritable,Writable,Writable>
+ implements SimpleEntityWriterConstants {
private static final Log LOG = LogFactory.getLog(SimpleEntityWriterV1.class);
- // constants for mtype = 1
- static final String KBS_SENT = "kbs sent";
- static final int KBS_SENT_DEFAULT = 1;
- static final String TEST_TIMES = "testtimes";
- static final int TEST_TIMES_DEFAULT = 100;
- static final String TIMELINE_SERVICE_PERFORMANCE_RUN_ID =
- "timeline.server.performance.run.id";
- /**
- * To ensure that the compression really gets exercised, generate a
- * random alphanumeric fixed length payload
- */
- private static char[] ALPHA_NUMS = new char[] { 'a', 'b', 'c', 'd', 'e', 'f',
- 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
- 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 'A', 'B', 'C', 'D',
- 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P',
- 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', '1', '2',
- '3', '4', '5', '6', '7', '8', '9', '0', ' ' };
-
public void map(IntWritable key, IntWritable val, Context context) throws IOException {
TimelineClient tlc = new TimelineClientImpl();
Configuration conf = context.getConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
new file mode 100644
index 0000000..d66deb0
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/SimpleEntityWriterV2.java
@@ -0,0 +1,131 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.TimelineServicePerformance.PerfCounters;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
+
+/**
+ * Adds simple entities with random string payload, events, metrics, and
+ * configuration.
+ */
+class SimpleEntityWriterV2 extends EntityWriterV2
+ implements SimpleEntityWriterConstants {
+ private static final Log LOG = LogFactory.getLog(SimpleEntityWriterV2.class);
+
+ protected void writeEntities(Configuration tlConf,
+ TimelineCollectorManager manager, Context context) throws IOException {
+ Configuration conf = context.getConfiguration();
+ // simulate the app id with the task id
+ int taskId = context.getTaskAttemptID().getTaskID().getId();
+ long timestamp = conf.getLong(TIMELINE_SERVICE_PERFORMANCE_RUN_ID, 0);
+ ApplicationId appId = ApplicationId.newInstance(timestamp, taskId);
+
+ // create the app level timeline collector
+ AppLevelTimelineCollector collector =
+ new AppLevelTimelineCollector(appId);
+ manager.putIfAbsent(appId, collector);
+
+ try {
+ // set the context
+ // flow id: job name, flow run id: timestamp, user id
+ TimelineCollectorContext tlContext =
+ collector.getTimelineEntityContext();
+ tlContext.setFlowName(context.getJobName());
+ tlContext.setFlowRunId(timestamp);
+ tlContext.setUserId(context.getUser());
+
+ final int kbs = conf.getInt(KBS_SENT, KBS_SENT_DEFAULT);
+
+ long totalTime = 0;
+ final int testtimes = conf.getInt(TEST_TIMES, TEST_TIMES_DEFAULT);
+ final Random rand = new Random();
+ final TaskAttemptID taskAttemptId = context.getTaskAttemptID();
+ final char[] payLoad = new char[kbs * 1024];
+
+ for (int i = 0; i < testtimes; i++) {
+ // Generate a fixed length random payload
+ for (int xx = 0; xx < kbs * 1024; xx++) {
+ int alphaNumIdx =
+ rand.nextInt(ALPHA_NUMS.length);
+ payLoad[xx] = ALPHA_NUMS[alphaNumIdx];
+ }
+ String entId = taskAttemptId + "_" + Integer.toString(i);
+ final TimelineEntity entity = new TimelineEntity();
+ entity.setId(entId);
+ entity.setType("FOO_ATTEMPT");
+ entity.addInfo("PERF_TEST", payLoad);
+ // add an event
+ TimelineEvent event = new TimelineEvent();
+ event.setId("foo_event_id");
+ event.setTimestamp(System.currentTimeMillis());
+ event.addInfo("foo_event", "test");
+ entity.addEvent(event);
+ // add a metric
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId("foo_metric");
+ metric.addValue(System.currentTimeMillis(), 123456789L);
+ entity.addMetric(metric);
+ // add a config
+ entity.addConfig("foo", "bar");
+
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(entity);
+ // use the current user for this purpose
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ long startWrite = System.nanoTime();
+ try {
+ collector.putEntities(entities, ugi);
+ } catch (Exception e) {
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).
+ increment(1);
+ LOG.error("writing to the timeline service failed", e);
+ }
+ long endWrite = System.nanoTime();
+ totalTime += TimeUnit.NANOSECONDS.toMillis(endWrite-startWrite);
+ }
+ LOG.info("wrote " + testtimes + " entities (" + kbs*testtimes +
+ " kB) in " + totalTime + " ms");
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).
+ increment(totalTime);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).
+ increment(testtimes);
+ context.getCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).
+ increment(kbs*testtimes);
+ } finally {
+ // clean up
+ manager.remove(appId);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
index 79d123e..4d8b74b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV1.java
@@ -25,11 +25,6 @@ import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
-import org.apache.hadoop.mapreduce.TaskAttemptID;
-import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
new file mode 100644
index 0000000..79633d2
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineEntityConverterV2.java
@@ -0,0 +1,211 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.mapreduce;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
+import org.apache.hadoop.mapreduce.TaskAttemptID;
+import org.apache.hadoop.mapreduce.TaskID;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskAttemptInfo;
+import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.TaskInfo;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+
+class TimelineEntityConverterV2 {
+ private static final Log LOG =
+ LogFactory.getLog(TimelineEntityConverterV2.class);
+
+ static final String JOB = "MAPREDUCE_JOB";
+ static final String TASK = "MAPREDUCE_TASK";
+ static final String TASK_ATTEMPT = "MAPREDUCE_TASK_ATTEMPT";
+
+ /**
+ * Creates job, task, and task attempt entities based on the job history info
+ * and configuration.
+ *
+ * Note: currently these are plan timeline entities created for mapreduce
+ * types. These are not meant to be the complete and accurate entity set-up
+ * for mapreduce jobs. We do not leverage hierarchical timeline entities. If
+ * we create canonical mapreduce hierarchical timeline entities with proper
+ * parent-child relationship, we could modify this to use that instead.
+ *
+ * Note that we also do not add info to the YARN application entity, which
+ * would be needed for aggregation.
+ */
+ public List<TimelineEntity> createTimelineEntities(JobInfo jobInfo,
+ Configuration conf) {
+ List<TimelineEntity> entities = new ArrayList<>();
+
+ // create the job entity
+ TimelineEntity job = createJobEntity(jobInfo, conf);
+ entities.add(job);
+
+ // create the task and task attempt entities
+ List<TimelineEntity> tasksAndAttempts =
+ createTaskAndTaskAttemptEntities(jobInfo);
+ entities.addAll(tasksAndAttempts);
+
+ return entities;
+ }
+
+ private TimelineEntity createJobEntity(JobInfo jobInfo, Configuration conf) {
+ TimelineEntity job = new TimelineEntity();
+ job.setType(JOB);
+ job.setId(jobInfo.getJobId().toString());
+ job.setCreatedTime(jobInfo.getSubmitTime());
+
+ job.addInfo("JOBNAME", jobInfo.getJobname());
+ job.addInfo("USERNAME", jobInfo.getUsername());
+ job.addInfo("JOB_QUEUE_NAME", jobInfo.getJobQueueName());
+ job.addInfo("SUBMIT_TIME", jobInfo.getSubmitTime());
+ job.addInfo("LAUNCH_TIME", jobInfo.getLaunchTime());
+ job.addInfo("FINISH_TIME", jobInfo.getFinishTime());
+ job.addInfo("JOB_STATUS", jobInfo.getJobStatus());
+ job.addInfo("PRIORITY", jobInfo.getPriority());
+ job.addInfo("TOTAL_MAPS", jobInfo.getTotalMaps());
+ job.addInfo("TOTAL_REDUCES", jobInfo.getTotalReduces());
+ job.addInfo("UBERIZED", jobInfo.getUberized());
+ job.addInfo("ERROR_INFO", jobInfo.getErrorInfo());
+
+ // add metrics from total counters
+ // we omit the map counters and reduce counters for now as it's kind of
+ // awkward to put them (map/reduce/total counters are really a group of
+ // related counters)
+ Counters totalCounters = jobInfo.getTotalCounters();
+ if (totalCounters != null) {
+ addMetrics(job, totalCounters);
+ }
+ // finally add configuration to the job
+ addConfiguration(job, conf);
+ LOG.info("converted job " + jobInfo.getJobId() + " to a timeline entity");
+ return job;
+ }
+
+ private void addConfiguration(TimelineEntity job, Configuration conf) {
+ for (Map.Entry<String,String> e: conf) {
+ job.addConfig(e.getKey(), e.getValue());
+ }
+ }
+
+ private void addMetrics(TimelineEntity entity, Counters counters) {
+ for (CounterGroup g: counters) {
+ String groupName = g.getName();
+ for (Counter c: g) {
+ String name = groupName + ":" + c.getName();
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(name);
+ metric.addValue(System.currentTimeMillis(), c.getValue());
+ entity.addMetric(metric);
+ }
+ }
+ }
+
+ private List<TimelineEntity> createTaskAndTaskAttemptEntities(
+ JobInfo jobInfo) {
+ List<TimelineEntity> entities = new ArrayList<>();
+ Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
+ LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
+ " tasks");
+ for (TaskInfo taskInfo: taskInfoMap.values()) {
+ TimelineEntity task = createTaskEntity(taskInfo);
+ entities.add(task);
+ // add the task attempts from this task
+ Set<TimelineEntity> taskAttempts = createTaskAttemptEntities(taskInfo);
+ entities.addAll(taskAttempts);
+ }
+ return entities;
+ }
+
+ private TimelineEntity createTaskEntity(TaskInfo taskInfo) {
+ TimelineEntity task = new TimelineEntity();
+ task.setType(TASK);
+ task.setId(taskInfo.getTaskId().toString());
+ task.setCreatedTime(taskInfo.getStartTime());
+
+ task.addInfo("START_TIME", taskInfo.getStartTime());
+ task.addInfo("FINISH_TIME", taskInfo.getFinishTime());
+ task.addInfo("TASK_TYPE", taskInfo.getTaskType());
+ task.addInfo("TASK_STATUS", taskInfo.getTaskStatus());
+ task.addInfo("ERROR_INFO", taskInfo.getError());
+
+ // add metrics from counters
+ Counters counters = taskInfo.getCounters();
+ if (counters != null) {
+ addMetrics(task, counters);
+ }
+ LOG.info("converted task " + taskInfo.getTaskId() +
+ " to a timeline entity");
+ return task;
+ }
+
+ private Set<TimelineEntity> createTaskAttemptEntities(TaskInfo taskInfo) {
+ Set<TimelineEntity> taskAttempts = new HashSet<TimelineEntity>();
+ Map<TaskAttemptID,TaskAttemptInfo> taskAttemptInfoMap =
+ taskInfo.getAllTaskAttempts();
+ LOG.info("task " + taskInfo.getTaskId() + " has " +
+ taskAttemptInfoMap.size() + " task attempts");
+ for (TaskAttemptInfo taskAttemptInfo: taskAttemptInfoMap.values()) {
+ TimelineEntity taskAttempt = createTaskAttemptEntity(taskAttemptInfo);
+ taskAttempts.add(taskAttempt);
+ }
+ return taskAttempts;
+ }
+
+ private TimelineEntity createTaskAttemptEntity(
+ TaskAttemptInfo taskAttemptInfo) {
+ TimelineEntity taskAttempt = new TimelineEntity();
+ taskAttempt.setType(TASK_ATTEMPT);
+ taskAttempt.setId(taskAttemptInfo.getAttemptId().toString());
+ taskAttempt.setCreatedTime(taskAttemptInfo.getStartTime());
+
+ taskAttempt.addInfo("START_TIME", taskAttemptInfo.getStartTime());
+ taskAttempt.addInfo("FINISH_TIME", taskAttemptInfo.getFinishTime());
+ taskAttempt.addInfo("MAP_FINISH_TIME",
+ taskAttemptInfo.getMapFinishTime());
+ taskAttempt.addInfo("SHUFFLE_FINISH_TIME",
+ taskAttemptInfo.getShuffleFinishTime());
+ taskAttempt.addInfo("SORT_FINISH_TIME",
+ taskAttemptInfo.getSortFinishTime());
+ taskAttempt.addInfo("TASK_STATUS", taskAttemptInfo.getTaskStatus());
+ taskAttempt.addInfo("STATE", taskAttemptInfo.getState());
+ taskAttempt.addInfo("ERROR", taskAttemptInfo.getError());
+ taskAttempt.addInfo("CONTAINER_ID",
+ taskAttemptInfo.getContainerId().toString());
+
+ // add metrics from counters
+ Counters counters = taskAttemptInfo.getCounters();
+ if (counters != null) {
+ addMetrics(taskAttempt, counters);
+ }
+ LOG.info("converted task attempt " + taskAttemptInfo.getAttemptId() +
+ " to a timeline entity");
+ return taskAttempt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
index 0753d7f..1a14137 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TimelineServicePerformance.java
@@ -23,8 +23,6 @@ import java.util.Date;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.SleepJob.SleepInputFormat;
import org.apache.hadoop.mapreduce.lib.output.NullOutputFormat;
import org.apache.hadoop.util.GenericOptionsParser;
@@ -46,15 +44,19 @@ public class TimelineServicePerformance extends Configured implements Tool {
System.err.println(
"Usage: [-m <maps>] number of mappers (default: " + NUM_MAPS_DEFAULT +
")\n" +
- " [-v] timeline service version\n" +
- " [-mtype <mapper type in integer>]\n" +
- " 1. simple entity write mapper (default)\n" +
+ " [-v] timeline service version (default: " +
+ TIMELINE_SERVICE_VERSION_1 + ")\n" +
+ " 1. version 1.x\n" +
+ " 2. version 2.x\n" +
+ " [-mtype <mapper type in integer>] (default: " +
+ SIMPLE_ENTITY_WRITER + ")\n" +
+ " 1. simple entity write mapper\n" +
" 2. jobhistory files replay mapper\n" +
" [-s <(KBs)test>] number of KB per put (mtype=1, default: " +
- SimpleEntityWriterV1.KBS_SENT_DEFAULT + " KB)\n" +
+ SimpleEntityWriterConstants.KBS_SENT_DEFAULT + " KB)\n" +
" [-t] package sending iterations per mapper (mtype=1, default: " +
- SimpleEntityWriterV1.TEST_TIMES_DEFAULT + ")\n" +
- " [-d <path>] root path of job history files (mtype=2)\n" +
+ SimpleEntityWriterConstants.TEST_TIMES_DEFAULT + ")\n" +
+ " [-d <path>] hdfs root path of job history files (mtype=2)\n" +
" [-r <replay mode>] (mtype=2)\n" +
" 1. write all entities for a job in one put (default)\n" +
" 2. write one entity at a time\n");
@@ -78,8 +80,7 @@ public class TimelineServicePerformance extends Configured implements Tool {
try {
if ("-v".equals(args[i])) {
timeline_service_version = Integer.parseInt(args[++i]);
- }
- if ("-m".equals(args[i])) {
+ } else if ("-m".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
job.getConfiguration()
.setInt(MRJobConfig.NUM_MAPS, Integer.parseInt(args[i]));
@@ -88,11 +89,12 @@ public class TimelineServicePerformance extends Configured implements Tool {
mapperType = Integer.parseInt(args[++i]);
} else if ("-s".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(SimpleEntityWriterV1.KBS_SENT, Integer.parseInt(args[i]));
+ conf.setInt(SimpleEntityWriterConstants.KBS_SENT,
+ Integer.parseInt(args[i]));
}
} else if ("-t".equals(args[i])) {
if (Integer.parseInt(args[++i]) > 0) {
- conf.setInt(SimpleEntityWriterV1.TEST_TIMES,
+ conf.setInt(SimpleEntityWriterConstants.TEST_TIMES,
Integer.parseInt(args[i]));
}
} else if ("-d".equals(args[i])) {
@@ -113,28 +115,40 @@ public class TimelineServicePerformance extends Configured implements Tool {
}
// handle mapper-specific settings
- switch (timeline_service_version) {
- case TIMELINE_SERVICE_VERSION_1:
- default:
- switch (mapperType) {
- case JOB_HISTORY_FILE_REPLAY_MAPPER:
+ switch (mapperType) {
+ case JOB_HISTORY_FILE_REPLAY_MAPPER:
+ String processingPath =
+ conf.get(JobHistoryFileReplayHelper.PROCESSING_PATH);
+ if (processingPath == null || processingPath.isEmpty()) {
+ System.out.println("processing path is missing while mtype = 2");
+ return printUsage() == 0;
+ }
+ switch (timeline_service_version) {
+ case TIMELINE_SERVICE_VERSION_2:
+ job.setMapperClass(JobHistoryFileReplayMapperV2.class);
+ break;
+ case TIMELINE_SERVICE_VERSION_1:
+ default:
job.setMapperClass(JobHistoryFileReplayMapperV1.class);
- String processingPath =
- conf.get(JobHistoryFileReplayHelper.PROCESSING_PATH);
- if (processingPath == null || processingPath.isEmpty()) {
- System.out.println("processing path is missing while mtype = 2");
- return printUsage() == 0;
- }
break;
- case SIMPLE_ENTITY_WRITER:
+ }
+ break;
+ case SIMPLE_ENTITY_WRITER:
+ default:
+ // use the current timestamp as the "run id" of the test: this will
+ // be used as simulating the cluster timestamp for apps
+ conf.setLong(SimpleEntityWriterConstants.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
+ System.currentTimeMillis());
+ switch (timeline_service_version) {
+ case TIMELINE_SERVICE_VERSION_2:
+ job.setMapperClass(SimpleEntityWriterV2.class);
+ break;
+ case TIMELINE_SERVICE_VERSION_1:
default:
job.setMapperClass(SimpleEntityWriterV1.class);
- // use the current timestamp as the "run id" of the test: this will
- // be used as simulating the cluster timestamp for apps
- conf.setLong(SimpleEntityWriterV1.TIMELINE_SERVICE_PERFORMANCE_RUN_ID,
- System.currentTimeMillis());
break;
}
+ break;
}
return true;
}
@@ -164,25 +178,46 @@ public class TimelineServicePerformance extends Configured implements Tool {
Date startTime = new Date();
System.out.println("Job started: " + startTime);
int ret = job.waitForCompletion(true) ? 0 : 1;
- org.apache.hadoop.mapreduce.Counters counters = job.getCounters();
- long writetime =
- counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_TIME).getValue();
- long writecounts =
- counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).getValue();
- long writesize =
- counters.findCounter(PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
- double transacrate = writecounts * 1000 / (double)writetime;
- double iorate = writesize * 1000 / (double)writetime;
- int numMaps =
- Integer.parseInt(job.getConfiguration().get(MRJobConfig.NUM_MAPS));
-
- System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
- " ops/s");
- System.out.println("IO RATE (per mapper): " + iorate + " KB/s");
-
- System.out.println("TRANSACTION RATE (total): " + transacrate*numMaps +
- " ops/s");
- System.out.println("IO RATE (total): " + iorate*numMaps + " KB/s");
+ if (job.isSuccessful()) {
+ org.apache.hadoop.mapreduce.Counters counters = job.getCounters();
+ long writecounts =
+ counters.findCounter(
+ PerfCounters.TIMELINE_SERVICE_WRITE_COUNTER).getValue();
+ long writefailures =
+ counters.findCounter(
+ PerfCounters.TIMELINE_SERVICE_WRITE_FAILURES).getValue();
+ if (writefailures > 0 && writefailures == writecounts) {
+ // see if we have a complete failure to write
+ System.out.println("Job failed: all writes failed!");
+ } else {
+ long writetime =
+ counters.findCounter(
+ PerfCounters.TIMELINE_SERVICE_WRITE_TIME).getValue();
+ long writesize =
+ counters.findCounter(
+ PerfCounters.TIMELINE_SERVICE_WRITE_KBS).getValue();
+ if (writetime == 0L) {
+ // see if write time is zero (normally shouldn't happen)
+ System.out.println("Job failed: write time is 0!");
+ } else {
+ double transacrate = writecounts * 1000 / (double)writetime;
+ double iorate = writesize * 1000 / (double)writetime;
+ int numMaps =
+ Integer.parseInt(
+ job.getConfiguration().get(MRJobConfig.NUM_MAPS));
+
+ System.out.println("TRANSACTION RATE (per mapper): " + transacrate +
+ " ops/s");
+ System.out.println("IO RATE (per mapper): " + iorate + " KB/s");
+
+ System.out.println("TRANSACTION RATE (total): " +
+ transacrate*numMaps + " ops/s");
+ System.out.println("IO RATE (total): " + iorate*numMaps + " KB/s");
+ }
+ }
+ } else {
+ System.out.println("Job failed: " + job.getStatus().getFailureInfo());
+ }
return ret;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: YARN-3150. Documenting the
timeline service v2. (Sangjin Lee and Vrushali C via gtcarrera9)
Posted by gt...@apache.org.
YARN-3150. Documenting the timeline service v2. (Sangjin Lee and Vrushali C via gtcarrera9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/caf23c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/caf23c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/caf23c34
Branch: refs/heads/YARN-2928
Commit: caf23c3453b7476df24edaaa024108cabcd56b21
Parents: 084a334
Author: Li Lu <gt...@apache.org>
Authored: Sat Apr 30 15:02:12 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:40 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +-
.../src/main/resources/yarn-default.xml | 6 +-
.../metrics/TimelineServiceV2Publisher.java | 16 +-
.../TestSystemMetricsPublisherForV2.java | 16 +-
.../src/site/markdown/TimelineServer.md | 2 +-
.../src/site/markdown/TimelineServiceV2.md | 576 +++++++++++++++++++
.../src/site/resources/images/timeline_v2.jpg | Bin 0 -> 45112 bytes
7 files changed, 600 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index b79b3a5..3837a2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -477,13 +477,13 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
/**
- * The setting that controls whether yarn container metrics is published to
- * the timeline server or not by RM. This configuration setting is for ATS
+ * The setting that controls whether yarn container events are published to
+ * the timeline service or not by RM. This configuration setting is for ATS
* V2
*/
- public static final String RM_PUBLISH_CONTAINER_METRICS_ENABLED = YARN_PREFIX
+ public static final String RM_PUBLISH_CONTAINER_EVENTS_ENABLED = YARN_PREFIX
+ "rm.system-metrics-publisher.emit-container-events";
- public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED =
+ public static final boolean DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED =
false;
public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 13d4b4f..4899dcb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -799,9 +799,9 @@
</property>
<property>
- <description>The setting that controls whether yarn container metrics is
- published to the timeline server or not by RM. This configuration setting is
- for ATS V2.</description>
+ <description>The setting that controls whether yarn container events are
+ published to the timeline service or not by RM. This configuration setting
+ is for ATS V2.</description>
<name>yarn.rm.system-metrics-publisher.emit-container-events</name>
<value>false</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 14073d1..b7ece5f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -68,7 +68,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
private static final Log LOG =
LogFactory.getLog(TimelineServiceV2Publisher.class);
private RMTimelineCollectorManager rmTimelineCollectorManager;
- private boolean publishContainerMetrics;
+ private boolean publishContainerEvents;
public TimelineServiceV2Publisher(RMContext rmContext) {
super("TimelineserviceV2Publisher");
@@ -80,14 +80,14 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
super.serviceInit(conf);
getDispatcher().register(SystemMetricsEventType.class,
new TimelineV2EventHandler());
- publishContainerMetrics = getConfig().getBoolean(
- YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
- YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
+ publishContainerEvents = getConfig().getBoolean(
+ YarnConfiguration.RM_PUBLISH_CONTAINER_EVENTS_ENABLED,
+ YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED);
}
@VisibleForTesting
- boolean isPublishContainerMetrics() {
- return publishContainerMetrics;
+ boolean isPublishContainerEvents() {
+ return publishContainerEvents;
}
@SuppressWarnings("unchecked")
@@ -285,7 +285,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
@SuppressWarnings("unchecked")
@Override
public void containerCreated(RMContainer container, long createdTime) {
- if (publishContainerMetrics) {
+ if (publishContainerEvents) {
TimelineEntity entity = createContainerEntity(container.getContainerId());
entity.setCreatedTime(createdTime);
@@ -320,7 +320,7 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
@SuppressWarnings("unchecked")
@Override
public void containerFinished(RMContainer container, long finishedTime) {
- if (publishContainerMetrics) {
+ if (publishContainerEvents) {
TimelineEntity entity = createContainerEntity(container.getContainerId());
TimelineEvent tEvent = new TimelineEvent();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 7a018ca..13c67f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -140,7 +140,7 @@ public class TestSystemMetricsPublisherForV2 {
conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
conf.setInt(
YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE, 2);
- conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
+ conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_EVENTS_ENABLED,
true);
try {
conf.set(FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_DIR_ROOT,
@@ -160,12 +160,12 @@ public class TestSystemMetricsPublisherForV2 {
new TimelineServiceV2Publisher(mock(RMContext.class));
try {
Configuration conf = getTimelineV2Conf();
- conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
- YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
+ conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_EVENTS_ENABLED,
+ YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_EVENTS_ENABLED);
metricsPublisher.init(conf);
assertFalse(
- "Default configuration should not publish container Metrics from RM",
- metricsPublisher.isPublishContainerMetrics());
+ "Default configuration should not publish container events from RM",
+ metricsPublisher.isPublishContainerEvents());
metricsPublisher.stop();
@@ -174,10 +174,10 @@ public class TestSystemMetricsPublisherForV2 {
metricsPublisher.init(conf);
assertTrue("Expected to have registered event handlers and set ready to "
+ "publish events after init",
- metricsPublisher.isPublishContainerMetrics());
+ metricsPublisher.isPublishContainerEvents());
metricsPublisher.start();
- assertTrue("Expected to publish container Metrics from RM",
- metricsPublisher.isPublishContainerMetrics());
+ assertTrue("Expected to publish container events from RM",
+ metricsPublisher.isPublishContainerEvents());
} finally {
metricsPublisher.stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
index f20bd2c..f09909b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServer.md
@@ -88,7 +88,7 @@ Current status
Future Plans
1. Future releases will introduce a next generation timeline service
-which is scalable and reliable, "Timeline Server v2".
+which is scalable and reliable, ["Timeline Service v2"](./TimelineServiceV2.html).
1. The expanded features of this service *may not* be available to
applications using the Timeline Server v1 REST API. That includes extended
data structures as well as the ability of the client to failover between Timeline Server instances.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
new file mode 100644
index 0000000..90db556
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/TimelineServiceV2.md
@@ -0,0 +1,576 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+The YARN Timeline Service v.2
+========================
+
+* [Overview](#Overview)
+ * [Introduction](#Introduction)
+ * [Architecture](#Architecture)
+ * [Current Status](#Current_Status)
+* [Deployment](#Deployment)
+ * [Configurations](#Configurations)
+ * [Enabling the Timeline Service v.2](#Enabling_Timeline_Service_v2)
+* [Publishing of application specific data](#Publishing_of_application_specific_data)
+* [Timeline Service v.2 REST API](#Timeline_Service_REST_API_v2)
+
+#<a name="Overview"></a>Overview
+
+### <a name="Introduction"></a>Introduction
+
+YARN Timeline Service v.2 is the next major iteration of Timeline Server, following v.1 and v.1.5.
+V.2 is created to address two major challenges of v.1.
+
+#### Scalability
+V.1 is limited to a single instance of writer/reader and storage, and does not scale well beyond
+small clusters. V.2 uses a more scalable distributed writer architecture and a scalable backend
+storage.
+
+YARN Timeline Service v.2 separates the collection (writes) of data from serving (reads) of data.
+It uses distributed collectors, essentially one collector for each YARN application. The readers
+are separate instances that are dedicated to serving queries via REST API.
+
+YARN Timeline Service v.2 chooses Apache HBase as the primary backing storage, as Apache HBase
+scales well to a large size while maintaining good response times for reads and writes.
+
+#### Usability improvements
+In many cases, users are interested in information at the level of "flows" or logical groups of
+YARN applications. It is much more common to launch a set or series of YARN applications to
+complete a logic application. Timeline Service v.2 supports the notion of flows explicitly. In
+addition, it supports aggregating metrics at the flow level.
+
+Also, information such as configuration and metrics is treated and supported as a first-class
+citizen.
+
+###<a name="Architecture"></a>Architecture
+
+YARN Timeline Service v.2 uses a set of collectors (writers) to write data to the backend storage.
+The collectors are distributed and co-located with the application masters to which they are
+dedicated. All data that belong to that application are sent to the application level timeline
+collectors with the exception of the resource manager timeline collector.
+
+For a given application, the application master can write data for the application to the
+co-located timeline collectors (which is an NM auxiliary service in this release). In addition,
+node managers of other nodes that are running the containers for the application also write data
+to the timeline collector on the node that is running the application master.
+
+The resource manager also maintains its own timeline collector. It emits only YARN-generic
+lifecycle events to keep its volume of writes reasonable.
+
+The timeline readers are separate daemons separate from the timeline collectors, and they are
+dedicated to serving queries via REST API.
+
+The following diagram illustrates the design at a high level.
+
+
+
+### <a name="Current_Status"></a>Current Status and Future Plans
+
+YARN Timeline Service v.2 is currently in alpha. It is very much work in progress, and many things
+can and will change rapidly. Users should enable Timeline Service v.2 only on a test or
+experimental cluster to test the feature.
+
+A complete end-to-end flow of writes and reads should be functional, with Apache HBase as the
+backend. You should be able to start generating data. When enabled, all YARN-generic events are
+published as well as YARN system metrics such as CPU and memory. Furthermore, some applications
+including Distributed Shell and MapReduce write per-framework data to YARN Timeline Service v.2.
+
+The REST API comes with a good number of useful and flexible query patterns (see below for more
+information).
+
+Although the basic mode of accessing data is via REST, it also comes with a basic web UI based on
+the proposed new YARN UI framework. Currently there is no support for command line access, however.
+
+The collectors (writers) are currently embedded in the node managers as auxiliary services. The
+resource manager also has its dedicated in-process collector. The reader is currently a single
+instance. Currently, it is not possible to write to Timeline Service outside the context of a YARN
+application (i.e. no off-cluster client).
+
+When YARN Timeline Service v.2 is disabled, one should expect no functional or performance impact
+on any other existing functionality.
+
+The work to make it production-ready continues. Some key items include
+
+* More robust storage fault tolerance
+* Security
+* Support for off-cluster clients
+* More complete and integrated web UI
+* Better support for long-running apps
+* Offline (time-based periodic) aggregation for flows, users, and queues for reporting and
+analysis
+* Timeline collectors as separate instances from node managers
+* Clustering of the readers
+* Migration and compatibility with v.1
+
+
+#<a name="Deployment"></a>Deployment
+
+###<a name="Configurations"></a>Configurations
+
+New configuration parameters that are introduced with v.2 are marked bold.
+
+#### Basic configuration
+
+| Configuration Property | Description |
+|:---- |:---- |
+| `yarn.timeline-service.enabled` | Indicate to clients whether Timeline service is enabled or not. If enabled, the `TimelineClient` library used by applications will post entities and events to the Timeline server. Defaults to `false`. |
+| `yarn.timeline-service.version` | Indicate what is the current version of the running timeline service. For example, if "yarn.timeline-service.version" is 1.5, and "yarn.timeline-service.enabled" is true, it means the cluster will and should bring up the timeline service v.1.5 (and nothing else). On the client side, if the client uses the same version of timeline service, it should succeed. If the client chooses to use a smaller version in spite of this, then depending on how robust the compatibility story is between versions, the results may vary. Defaults to `1.0f`. |
+| **`yarn.timeline-service.writer.class`** | The class for the backend storage writer. Defaults to a filesystem storage writer, therefore it should be overridden. |
+| **`yarn.timeline-service.reader.class`** | The class for the backend storage reader. Defaults to a filesystem storage reader, therefore it should be overridden. |
+| **`yarn.system-metrics-publisher.enabled`** | The setting that controls whether yarn system metrics is published on the Timeline service or not by RM And NM. Defaults to `false`. |
+| **`yarn.rm.system-metrics-publisher.emit-container-events`** | The setting that controls whether yarn container metrics is published to the timeline server or not by RM. This configuration setting is for ATS V2. Defaults to `false`. |
+
+#### Advanced configuration
+
+| Configuration Property | Description |
+|:---- |:---- |
+| `yarn.timeline-service.hostname` | The hostname of the Timeline service web application. Defaults to `0.0.0.0` |
+| `yarn.timeline-service.address` | Address for the Timeline server to start the RPC server. Defaults to `${yarn.timeline-service.hostname}:10200`. |
+| `yarn.timeline-service.webapp.address` | The http address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8188`. |
+| `yarn.timeline-service.webapp.https.address` | The https address of the Timeline service web application. Defaults to `${yarn.timeline-service.hostname}:8190`. |
+| **`yarn.timeline-service.writer.flush-interval-seconds`** | The setting that controls how often the timeline collector flushes the timeline writer. Defaults to `60`. |
+| **`yarn.timeline-service.app-collector.linger-period.ms`** | Time period till which the application collector will be alive in NM, after the application master container finishes. Defaults to `1000` (1 second). |
+| **`yarn.timeline-service.timeline-client.number-of-async-entities-to-merge`** | Time line V2 client tries to merge these many number of async entities (if available) and then call the REST ATS V2 API to submit. Defaults to `10`. |
+| **`yarn.timeline-service.coprocessor.app-final-value-retention-milliseconds`** | The setting that controls how long the final value of a metric of a completed app is retained before merging into the flow sum. Defaults to `259200000` (3 days). |
+
+
+
+### <a name="Enabling_Timeline_Service_v2"></a>Enabling the Timeline Service v.2
+
+#### Preparing Apache HBase cluster for storage
+The first part is to set up or pick an Apache HBase cluster to use as the storage cluster. Once
+you have an HBase cluster ready to use for this purpose, perform the following steps.
+
+First, add the timeline service jar to the HBase classpath in all HBase machines in the cluster. It
+is needed for the coprocessor as well as the schema creator. For example,
+
+ cp hadoop-yarn-server-timelineservice-3.0.0-SNAPSHOT.jar /usr/hbase/lib/
+
+Then, enable the coprocessor that handles the aggregation. To enable it, add the following entry in
+region servers' `hbase-site.xml` file (generally located in the `conf` directory) as follows:
+
+```
+<property>
+ <name>hbase.coprocessor.region.classes</name>
+ <value>org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunCoprocessor</value>
+</property>
+```
+
+Restart the region servers and the master to pick up the timeline service jar as well as the config
+change. In this version, the coprocessor is loaded statically (i.e. system coprocessor) as opposed
+to a dynamically (table coprocessor).
+
+Finally, run the schema creator tool to create the necessary tables:
+
+ bin/hbase org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator
+
+The `TimelineSchemaCreator` tool supports a few options that may come handy especially when you
+are testing. For example, you can use `-skipExistingTable` (`-s` for short) to skip existing tables
+and continue to create other tables rather than failing the schema creation.
+
+#### Enabling Timeline Service v.2
+Following are the basic configurations to start Timeline service v.2:
+
+```
+<property>
+ <name>yarn.timeline-service.version</name>
+ <value>2.0f</value>
+</property>
+
+<property>
+ <name>yarn.timeline-service.enabled</name>
+ <value>true</value>
+</property>
+
+<property>
+ <name>yarn.timeline-service.writer.class</name>
+ <value>org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl</value>
+</property>
+
+<property>
+ <name>yarn.timeline-service.reader.class</name>
+ <value>org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl</value>
+</property>
+
+<property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle,timeline_collector</value>
+</property>
+
+<property>
+ <name>yarn.nodemanager.aux-services.timeline_collector.class</name>
+ <value>org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService</value>
+</property>
+
+<property>
+ <description>The setting that controls whether yarn system metrics is
+ published on the Timeline service or not by RM And NM.</description>
+ <name>yarn.system-metrics-publisher.enabled</name>
+ <value>true</value>
+</property>
+
+<property>
+ <description>The setting that controls whether yarn container events are
+ published to the timeline service or not by RM. This configuration setting
+ is for ATS V2.</description>
+ <name>yarn.rm.system-metrics-publisher.emit-container-events</name>
+ <value>true</value>
+</property>
+```
+
+In addition, you may want to set the YARN cluster name to a reasonably unique name in case you
+are using multiple clusters to store data in the same Apache HBase storage:
+
+```
+<property>
+ <name>yarn.resourcemanager.cluster-id</name>
+ <value>my_research_test_cluster</value>
+</property>
+```
+
+Also, add the `hbase-site.xml` configuration file to the client Hadoop cluster configuration so
+that it can write data to the Apache HBase cluster you are using.
+
+#### Running Timeline Service v.2
+Restart the resource manager as well as the node managers to pick up the new configuration. The
+collectors start within the resource manager and the node managers in an embedded manner.
+
+The Timeline Service reader is a separate YARN daemon, and it can be started using the following
+syntax:
+
+ $ yarn-daemon.sh start timelinereader
+
+#### Enabling MapReduce to write to Timeline Service v.2
+To write MapReduce framework data to Timeline Service v.2, enable the following configuration in
+`mapred-site.xml`:
+
+```
+<property>
+ <name>mapreduce.job.emit-timeline-data</name>
+ <value>true</value>
+</property>
+```
+
+###<a name="Publishing_of_application_specific_data"></a> Publishing application specific data
+
+This section is for YARN application developers that want to integrate with Timeline Service v.2.
+
+Developers can continue to use the `TimelineClient` API to publish per-framework data to the
+Timeline Service v.2. You only need to instantiate the right type of the client to write to v.2.
+On the other hand, the entity/object API for v.2 is different than v.1 as the object model is
+significantly changed. The v.2 timeline entity class is `org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity`
+whereas the v.1 class is `org.apache.hadoop.yarn.api.records.timeline.TimelineEntity`. The methods
+on `TimelineClient` suitable for writing to the Timeline Service v.2 are clearly delineated, and
+they use the v.2 types as arguments.
+
+Timeline Service v.2 `putEntities` methods come in 2 varieties: `putEntities` and
+`putEntitiesAsync`. The former is a blocking operation which should be used for writing more
+critical data (e.g. lifecycle events). The latter is a non-blocking operation. Note that neither
+has a return value.
+
+Creating a `TimelineClient` for v.2 involves passing in the application id to the factory method.
+
+For example:
+
+
+ // Create and start the Timeline client v.2
+ TimelineClient client = TimelineClient.createTimelineClient(appId);
+ client.init(conf);
+ client.start();
+
+ try {
+ TimelineEntity myEntity = new TimelineEntity();
+ myEntity.setEntityType("MY_APPLICATION");
+ myEntity.setEntityId("MyApp1")
+ // Compose other entity info
+
+ // Blocking write
+ client.putEntities(entity);
+
+ TimelineEntity myEntity2 = new TimelineEntity();
+ // Compose other info
+
+ // Non-blocking write
+ timelineClient.putEntitiesAsync(entity);
+
+ } catch (IOException e) {
+ // Handle the exception
+ } catch (RuntimeException e) {
+ // In Hadoop 2.6, if attempts submit information to the Timeline Server fail more than the retry limit,
+ // a RuntimeException will be raised. This may change in future releases, being
+ // replaced with a IOException that is (or wraps) that which triggered retry failures.
+ } catch (YarnException e) {
+ // Handle the exception
+ } finally {
+ // Stop the Timeline client
+ client.stop();
+ }
+
+As evidenced above, you need to specify the YARN application id to be able to write to the Timeline
+Service v.2. Note that currently you need to be on the cluster to be able to write to the Timeline
+Service. For example, an application master or code in the container can write to the Timeline
+Service, while an off-cluster MapReduce job submitter cannot.
+
+You can create and publish your own entities, events, and metrics as with previous versions.
+
+Application frameworks should set the "flow context" whenever possible in order to take advantage
+of the flow support Timeline Service v.2 provides. The flow context consists of the following:
+
+* Flow name: a string that identifies the high-level flow (e.g. "distributed grep" or any
+identifiable name that can uniquely represent the app)
+* Flow run id: a monotonically-increasing sequence of numbers that distinguish different runs of
+the same flow
+* (optional) Flow version: a string identifier that denotes a version of the flow
+
+If the flow context is not specified, defaults are supplied for these attributes:
+
+* Flow name: the YARN application name (or the application id if the name is not set)
+* Flow run id: the application start time in Unix time (milliseconds)
+* Flow version: "1"
+
+You can provide the flow context via YARN application tags:
+
+ ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext();
+
+ // set the flow context as YARN application tags
+ Set<String> tags = new HashSet<>();
+ tags.add(TimelineUtils.generateFlowNameTag("distributed grep"));
+ tags.add(Timelineutils.generateFlowVersionTag("3df8b0d6100530080d2e0decf9e528e57c42a90a"));
+ tags.add(TimelineUtils.generateFlowRunIdTag(System.currentTimeMillis()));
+
+ appContext.setApplicationTags(tags);
+
+# <a name="Timeline_Service_REST_API_v2"></a>Timeline Service v.2 REST API
+
+Querying the Timeline Service v.2 is currently only supported via REST API; there is no API
+client implemented in the YARN libraries.
+
+The v.2 REST API is implemented at under the path, `/ws/v2/timeline/` on the Timeline Service web
+service.
+
+Here is an informal description of the API.
+
+### Root path
+
+ GET /ws/v2/timeline/
+
+Returns a JSON object describing the service instance and version information.
+
+ {
+ "About":"Timeline Reader API",
+ "timeline-service-version":"3.0.0-SNAPSHOT",
+ "timeline-service-build-version":"3.0.0-SNAPSHOT from fb0acd08e6f0b030d82eeb7cbfa5404376313e60 by sjlee source checksum be6cba0e42417d53be16459e1685e7",
+ "timeline-service-version-built-on":"2016-04-11T23:15Z",
+ "hadoop-version":"3.0.0-SNAPSHOT",
+ "hadoop-build-version":"3.0.0-SNAPSHOT from fb0acd08e6f0b030d82eeb7cbfa5404376313e60 by sjlee source checksum ee968fd0aedcc7384230ee3ca216e790",
+ "hadoop-version-built-on":"2016-04-11T23:14Z"
+ }
+
+### Request Examples
+
+The following shows some of the supported queries on the REST API. For example, to get the most
+recent flow activities,
+
+HTTP request:
+
+ GET /ws/v2/timeline/clusters/{cluster name}/flows/
+
+Response:
+
+ [
+ {
+ "metrics": [],
+ "events": [],
+ "id": "test-cluster/1460419200000/sjlee@ds-date",
+ "type": "YARN_FLOW_ACTIVITY",
+ "createdtime": 0,
+ "flowruns": [
+ {
+ "metrics": [],
+ "events": [],
+ "id": "sjlee@ds-date/1460420305659",
+ "type": "YARN_FLOW_RUN",
+ "createdtime": 0,
+ "info": {
+ "SYSTEM_INFO_FLOW_VERSION": "1",
+ "SYSTEM_INFO_FLOW_RUN_ID": 1460420305659,
+ "SYSTEM_INFO_FLOW_NAME": "ds-date",
+ "SYSTEM_INFO_USER": "sjlee"
+ },
+ "isrelatedto": {},
+ "relatesto": {}
+ },
+ {
+ "metrics": [],
+ "events": [],
+ "id": "sjlee@ds-date/1460420587974",
+ "type": "YARN_FLOW_RUN",
+ "createdtime": 0,
+ "info": {
+ "SYSTEM_INFO_FLOW_VERSION": "1",
+ "SYSTEM_INFO_FLOW_RUN_ID": 1460420587974,
+ "SYSTEM_INFO_FLOW_NAME": "ds-date",
+ "SYSTEM_INFO_USER": "sjlee"
+ },
+ "isrelatedto": {},
+ "relatesto": {}
+ }
+ ],
+ "info": {
+ "SYSTEM_INFO_CLUSTER": "test-cluster",
+ "UID": "test-cluster!sjlee!ds-date",
+ "SYSTEM_INFO_FLOW_NAME": "ds-date",
+ "SYSTEM_INFO_DATE": 1460419200000,
+ "SYSTEM_INFO_USER": "sjlee"
+ },
+ "isrelatedto": {},
+ "relatesto": {}
+ }
+ ]
+
+It returns the flows that had runs (specific instances of the flows) most recently.
+
+You can drill further down to get the runs (specific instances) of a given flow.
+
+HTTP request:
+
+ GET /ws/v2/timeline/users/{user name}/flows/{flow name}/runs/
+
+Response:
+
+ [
+ {
+ "metrics": [],
+ "events": [],
+ "id": "sjlee@ds-date/1460420587974",
+ "type": "YARN_FLOW_RUN",
+ "createdtime": 1460420587974,
+ "info": {
+ "UID": "test-cluster!sjlee!ds-date!1460420587974",
+ "SYSTEM_INFO_FLOW_RUN_ID": 1460420587974,
+ "SYSTEM_INFO_FLOW_NAME": "ds-date",
+ "SYSTEM_INFO_FLOW_RUN_END_TIME": 1460420595198,
+ "SYSTEM_INFO_USER": "sjlee"
+ },
+ "isrelatedto": {},
+ "relatesto": {}
+ },
+ {
+ "metrics": [],
+ "events": [],
+ "id": "sjlee@ds-date/1460420305659",
+ "type": "YARN_FLOW_RUN",
+ "createdtime": 1460420305659,
+ "info": {
+ "UID": "test-cluster!sjlee!ds-date!1460420305659",
+ "SYSTEM_INFO_FLOW_RUN_ID": 1460420305659,
+ "SYSTEM_INFO_FLOW_NAME": "ds-date",
+ "SYSTEM_INFO_FLOW_RUN_END_TIME": 1460420311966,
+ "SYSTEM_INFO_USER": "sjlee"
+ },
+ "isrelatedto": {},
+ "relatesto": {}
+ }
+ ]
+
+This returns the most recent runs that belong to the given flow.
+
+You can provide a `limit` query parameter to limit the number of entries that returned in a query.
+If you want to limit the number of flow runs in the above query, you can do the following:
+
+HTTP request:
+
+ GET /ws/v2/timeline/users/{user name}/flows/{flow name}/runs?limit=1
+
+Response:
+
+ [
+ {
+ "metrics": [],
+ "events": [],
+ "id": "sjlee@ds-date/1460420587974",
+ "type": "YARN_FLOW_RUN",
+ "createdtime": 1460420587974,
+ "info": {
+ "UID": "test-cluster!sjlee!ds-date!1460420587974",
+ "SYSTEM_INFO_FLOW_RUN_ID": 1460420587974,
+ "SYSTEM_INFO_FLOW_NAME": "ds-date",
+ "SYSTEM_INFO_FLOW_RUN_END_TIME": 1460420595198,
+ "SYSTEM_INFO_USER": "sjlee"
+ },
+ "isrelatedto": {},
+ "relatesto": {}
+ }
+ ]
+
+Most queries in the v.2 REST API support the following query parameters:
+
+* `limit`
+* `createdtimestart`
+* `createdtimeend`
+* `relatesto`
+* `isrelatedto`
+* `infofilters`
+* `conffilters`
+* `metricfilters`
+* `eventfilters`
+* `fields`
+
+Given a flow run, you can query all the YARN applications that are part of that flow run:
+
+HTTP request:
+
+ GET /ws/v2/timeline/users/{user name}/flows/{flow name}/runs/{run id}/apps/
+
+Response:
+
+ [
+ {
+ "metrics": [],
+ "events": [],
+ "id": "application_1460419579913_0002",
+ "type": "YARN_APPLICATION",
+ "createdtime": 0,
+ "info": {
+ "UID": "test-cluster!sjlee!ds-date!1460420587974!application_1460419579913_0002"
+ },
+ "configs": {},
+ "isrelatedto": {},
+ "relatesto": {}
+ }
+ ]
+
+You can also provide per-framework entity type to query for them. For example,
+
+HTTP request:
+
+ GET /ws/v2/timeline/clusters/{cluster name}/apps/{app id}/entities/DS_APP_ATTEMPT
+
+Response:
+
+ [
+ {
+ "metrics": [],
+ "events": [],
+ "id": "appattempt_1460419579913_0002_000001",
+ "type": "DS_APP_ATTEMPT",
+ "createdtime": 0,
+ "info": {
+ "UID": "test-cluster!application_1460419579913_0002!DS_APP_ATTEMPT!appattempt_1460419579913_0002_000001"
+ },
+ "configs": {},
+ "isrelatedto": {},
+ "relatesto": {}
+ }
+ ]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/caf23c34/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/timeline_v2.jpg
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/timeline_v2.jpg b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/timeline_v2.jpg
new file mode 100644
index 0000000..a356e26
Binary files /dev/null and b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/resources/images/timeline_v2.jpg differ
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters
in TimelineReader (Varun Saxena via sjlee)
Posted by gt...@apache.org.
YARN-3863. Support complex filters in TimelineReader (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b2df86c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b2df86c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b2df86c
Branch: refs/heads/YARN-2928
Commit: 2b2df86c775f008c258fe128d3686f4a4d3b1a52
Parents: 408f001
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Apr 11 21:07:32 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:33 2016 -0700
----------------------------------------------------------------------
.../reader/TimelineEntityFilters.java | 170 +-
.../reader/TimelineReaderWebServicesUtils.java | 88 +-
.../reader/filter/TimelineCompareFilter.java | 35 +-
.../reader/filter/TimelineExistsFilter.java | 62 +
.../reader/filter/TimelineFilter.java | 16 +-
.../reader/filter/TimelineFilterList.java | 14 +
.../reader/filter/TimelineFilterUtils.java | 206 +-
.../reader/filter/TimelineKeyValueFilter.java | 48 +
.../reader/filter/TimelineKeyValuesFilter.java | 71 +
.../reader/filter/TimelinePrefixFilter.java | 6 +
.../storage/FileSystemTimelineReaderImpl.java | 36 +-
.../storage/HBaseTimelineWriterImpl.java | 31 +-
.../storage/application/ApplicationColumn.java | 28 +-
.../application/ApplicationColumnPrefix.java | 37 +-
.../storage/apptoflow/AppToFlowColumn.java | 16 +
.../timelineservice/storage/common/Column.java | 17 +
.../storage/common/ColumnHelper.java | 16 +
.../storage/common/ColumnPrefix.java | 35 +
.../common/TimelineEntityFiltersType.java | 71 +
.../storage/common/TimelineStorageUtils.java | 461 +++-
.../storage/entity/EntityColumn.java | 28 +-
.../storage/entity/EntityColumnPrefix.java | 38 +-
.../storage/flow/FlowActivityColumnPrefix.java | 35 +
.../storage/flow/FlowRunColumn.java | 3 +
.../storage/flow/FlowRunColumnPrefix.java | 26 +
.../storage/flow/FlowScanner.java | 1 +
.../storage/reader/ApplicationEntityReader.java | 426 ++--
.../reader/FlowActivityEntityReader.java | 7 +
.../storage/reader/FlowRunEntityReader.java | 97 +-
.../storage/reader/GenericEntityReader.java | 623 ++---
.../storage/reader/TimelineEntityReader.java | 71 +-
.../reader/TestTimelineReaderWebServices.java | 10 +-
.../TestFileSystemTimelineReaderImpl.java | 332 ++-
.../storage/TestHBaseTimelineStorage.java | 2172 +++++++++++++++++-
.../storage/flow/TestFlowDataGenerator.java | 16 +
.../storage/flow/TestHBaseStorageFlowRun.java | 267 ++-
.../flow/TestHBaseStorageFlowRunCompaction.java | 2 +-
.../reader/filter/TimelineExistsFilter.java | 62 +
.../reader/filter/TimelineKeyValueFilter.java | 48 +
.../reader/filter/TimelineKeyValuesFilter.java | 71 +
.../common/TimelineEntityFiltersType.java | 71 +
41 files changed, 5054 insertions(+), 816 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
index 5b2c300..4821d31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
@@ -18,11 +18,14 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
/**
* Encapsulates information regarding the filters to apply while querying. These
@@ -36,36 +39,81 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
* <li><b>createdTimeEnd</b> - Matched entities should not be created after
* this timestamp. If null or {@literal <=0}, defaults to
* {@link Long#MAX_VALUE}.</li>
- * <li><b>relatesTo</b> - Matched entities should relate to given entities.
- * If null or empty, the relations are not matched.</li>
- * <li><b>isRelatedTo</b> - Matched entities should be related to given
- * entities. If null or empty, the relations are not matched.</li>
+ * <li><b>relatesTo</b> - Matched entities should or should not relate to given
+ * entities depending on what's specified in the filter. The entities in
+ * relatesTo are identified by entity type and id. This is represented as
+ * a {@link TimelineFilterList} object containing
+ * {@link TimelineKeyValuesFilter} objects, each of which contains a
+ * set of values for a key and the comparison operator (equals/not equals). The
+ * key which represents the entity type is a string and values are a set of
+ * entity identifiers (also string). As it is a filter list, relatesTo can be
+ * evaluated with logical AND/OR and we can create a hierarchy of these
+ * {@link TimelineKeyValuesFilter} objects. If null or empty, the relations are
+ * not matched.</li>
+ * <li><b>isRelatedTo</b> - Matched entities should or should not be related
+ * to given entities depending on what's specified in the filter. The entities
+ * in isRelatedTo are identified by entity type and id. This is represented as
+ * a {@link TimelineFilterList} object containing
+ * {@link TimelineKeyValuesFilter} objects, each of which contains a
+ * set of values for a key and the comparison operator (equals/not equals). The
+ * key which represents the entity type is a string and values are a set of
+ * entity identifiers (also string). As it is a filter list, relatesTo can be
+ * evaluated with logical AND/OR and we can create a hierarchy of these
+ * {@link TimelineKeyValuesFilter} objects. If null or empty, the relations are
+ * not matched.</li>
* <li><b>infoFilters</b> - Matched entities should have exact matches to
- * the given info represented as key-value pairs. If null or empty, the
- * filter is not applied.</li>
+ * the given info and should be either equal or not equal to given value
+ * depending on what's specified in the filter. This is represented as a
+ * {@link TimelineFilterList} object containing {@link TimelineKeyValueFilter}
+ * objects, each of which contains key-value pairs with a comparison operator
+ * (equals/not equals). The key which represents the info key is a string but
+ * value can be any object. As it is a filter list, info filters can be
+ * evaluated with logical AND/OR and we can create a hierarchy of these
+ * key-value pairs. If null or empty, the filter is not applied.</li>
* <li><b>configFilters</b> - Matched entities should have exact matches to
- * the given configs represented as key-value pairs. If null or empty, the
- * filter is not applied.</li>
+ * the given configurations and should be either equal or not equal to given
+ * value depending on what's specified in the filter. This is represented as a
+ * {@link TimelineFilterList} object containing {@link TimelineKeyValueFilter}
+ * objects, each of which contains key-value pairs with a comparison operator
+ * (equals/not equals). Both key (which represents config name) and value (which
+ * is config value) are strings. As it is a filter list, config filters can be
+ * evaluated with logical AND/OR and we can create a hierarchy of these
+ * {@link TimelineKeyValueFilter} objects. If null or empty, the filter is not
+ * applied.</li>
* <li><b>metricFilters</b> - Matched entities should contain the given
- * metrics. If null or empty, the filter is not applied.</li>
- * <li><b>eventFilters</b> - Matched entities should contain the given
- * events. If null or empty, the filter is not applied.</li>
+ * metrics and satisfy the specified relation with the value. This is
+ * represented as a {@link TimelineFilterList} object containing
+ * {@link TimelineCompareFilter} objects, each of which contains key-value pairs
+ * along with the specified relational/comparison operator represented by
+ * {@link TimelineCompareOp}. The key is a string and value is integer
+ * (Short/Integer/Long). As it is a filter list, metric filters can be evaluated
+ * with logical AND/OR and we can create a hierarchy of these
+ * {@link TimelineCompareFilter} objects. If null or empty, the filter is not
+ * applied.</li>
+ * <li><b>eventFilters</b> - Matched entities should contain or not contain the
+ * given events. This is represented as a {@link TimelineFilterList} object
+ * containing {@link TimelineExistsFilter} objects, each of which contains a
+ * value which must or must not exist depending on comparison operator specified
+ * in the filter. For event filters, the value represents a event id. As it is a
+ * filter list, event filters can be evaluated with logical AND/OR and we can
+ * create a hierarchy of these {@link TimelineExistsFilter} objects. If null or
+ * empty, the filter is not applied.</li>
* </ul>
*/
@Private
@Unstable
public class TimelineEntityFilters {
- private Long limit;
- private Long createdTimeBegin;
- private Long createdTimeEnd;
- private Map<String, Set<String>> relatesTo;
- private Map<String, Set<String>> isRelatedTo;
- private Map<String, Object> infoFilters;
- private Map<String, String> configFilters;
- private Set<String> metricFilters;
- private Set<String> eventFilters;
- private static final Long DEFAULT_BEGIN_TIME = 0L;
- private static final Long DEFAULT_END_TIME = Long.MAX_VALUE;
+ private long limit;
+ private long createdTimeBegin;
+ private long createdTimeEnd;
+ private TimelineFilterList relatesTo;
+ private TimelineFilterList isRelatedTo;
+ private TimelineFilterList infoFilters;
+ private TimelineFilterList configFilters;
+ private TimelineFilterList metricFilters;
+ private TimelineFilterList eventFilters;
+ private static final long DEFAULT_BEGIN_TIME = 0L;
+ private static final long DEFAULT_END_TIME = Long.MAX_VALUE;
/**
* Default limit of number of entities to return for getEntities API.
@@ -78,23 +126,26 @@ public class TimelineEntityFilters {
public TimelineEntityFilters(
Long entityLimit, Long timeBegin, Long timeEnd,
- Map<String, Set<String>> entityRelatesTo,
- Map<String, Set<String>> entityIsRelatedTo,
- Map<String, Object> entityInfoFilters,
- Map<String, String> entityConfigFilters,
- Set<String> entityMetricFilters,
- Set<String> entityEventFilters) {
- this.limit = entityLimit;
- if (this.limit == null || this.limit < 0) {
+ TimelineFilterList entityRelatesTo,
+ TimelineFilterList entityIsRelatedTo,
+ TimelineFilterList entityInfoFilters,
+ TimelineFilterList entityConfigFilters,
+ TimelineFilterList entityMetricFilters,
+ TimelineFilterList entityEventFilters) {
+ if (entityLimit == null || entityLimit < 0) {
this.limit = DEFAULT_LIMIT;
+ } else {
+ this.limit = entityLimit;
}
- this.createdTimeBegin = timeBegin;
- if (this.createdTimeBegin == null || this.createdTimeBegin < 0) {
+ if (timeBegin == null || timeBegin < 0) {
this.createdTimeBegin = DEFAULT_BEGIN_TIME;
+ } else {
+ this.createdTimeBegin = timeBegin;
}
- this.createdTimeEnd = timeEnd;
- if (this.createdTimeEnd == null || this.createdTimeEnd < 0) {
+ if (timeEnd == null || timeEnd < 0) {
this.createdTimeEnd = DEFAULT_END_TIME;
+ } else {
+ this.createdTimeEnd = timeEnd;
}
this.relatesTo = entityRelatesTo;
this.isRelatedTo = entityIsRelatedTo;
@@ -104,84 +155,87 @@ public class TimelineEntityFilters {
this.eventFilters = entityEventFilters;
}
- public Long getLimit() {
+ public long getLimit() {
return limit;
}
public void setLimit(Long entityLimit) {
- this.limit = entityLimit;
- if (this.limit == null || this.limit < 0) {
+ if (entityLimit == null || entityLimit < 0) {
this.limit = DEFAULT_LIMIT;
+ } else {
+ this.limit = entityLimit;
}
}
- public Long getCreatedTimeBegin() {
+ public long getCreatedTimeBegin() {
return createdTimeBegin;
}
public void setCreatedTimeBegin(Long timeBegin) {
- this.createdTimeBegin = timeBegin;
- if (this.createdTimeBegin == null || this.createdTimeBegin < 0) {
+ if (timeBegin == null || timeBegin < 0) {
this.createdTimeBegin = DEFAULT_BEGIN_TIME;
+ } else {
+ this.createdTimeBegin = timeBegin;
}
}
- public Long getCreatedTimeEnd() {
+ public long getCreatedTimeEnd() {
return createdTimeEnd;
}
public void setCreatedTimeEnd(Long timeEnd) {
- this.createdTimeEnd = timeEnd;
- if (this.createdTimeEnd == null || this.createdTimeEnd < 0) {
+ if (timeEnd == null || timeEnd < 0) {
this.createdTimeEnd = DEFAULT_END_TIME;
+ } else {
+ this.createdTimeEnd = timeEnd;
}
}
- public Map<String, Set<String>> getRelatesTo() {
+ public TimelineFilterList getRelatesTo() {
return relatesTo;
}
- public void setRelatesTo(Map<String, Set<String>> relations) {
+ public void setRelatesTo(TimelineFilterList relations) {
this.relatesTo = relations;
}
- public Map<String, Set<String>> getIsRelatedTo() {
+ public TimelineFilterList getIsRelatedTo() {
return isRelatedTo;
}
- public void setIsRelatedTo(Map<String, Set<String>> relations) {
+ public void setIsRelatedTo(TimelineFilterList relations) {
this.isRelatedTo = relations;
}
- public Map<String, Object> getInfoFilters() {
+ public TimelineFilterList getInfoFilters() {
return infoFilters;
}
- public void setInfoFilters(Map<String, Object> filters) {
+ public void setInfoFilters(TimelineFilterList filters) {
this.infoFilters = filters;
}
- public Map<String, String> getConfigFilters() {
+ public TimelineFilterList getConfigFilters() {
return configFilters;
}
- public void setConfigFilters(Map<String, String> filters) {
+ public void setConfigFilters(TimelineFilterList filters) {
this.configFilters = filters;
}
- public Set<String> getMetricFilters() {
+ public TimelineFilterList getMetricFilters() {
return metricFilters;
}
- public void setMetricFilters(Set<String> filters) {
+ public void setMetricFilters(TimelineFilterList filters) {
this.metricFilters = filters;
}
- public Set<String> getEventFilters() {
+ public TimelineFilterList getEventFilters() {
return eventFilters;
}
- public void setEventFilters(Set<String> filters) {
+ public void setEventFilters(TimelineFilterList filters) {
this.eventFilters = filters;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index d12f7e5..57d75db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -20,15 +20,19 @@ package org.apache.hadoop.yarn.server.timelineservice.reader;
import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
import javax.servlet.http.HttpServletRequest;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
/**
@@ -87,7 +91,7 @@ final class TimelineReaderWebServicesUtils {
parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
- parseValuesStr(metricfilters, COMMA_DELIMITER),
+ parseMetricFilters(metricfilters, COMMA_DELIMITER),
parseValuesStr(eventfilters, COMMA_DELIMITER));
}
@@ -114,22 +118,26 @@ final class TimelineReaderWebServicesUtils {
* @param delimiter string is delimited by this delimiter.
* @return set of strings.
*/
- static Set<String> parseValuesStr(String str, String delimiter) {
+ static TimelineFilterList parseValuesStr(String str, String delimiter) {
if (str == null || str.isEmpty()) {
return null;
}
- Set<String> strSet = new HashSet<String>();
+ TimelineFilterList filterList = new TimelineFilterList();
String[] strs = str.split(delimiter);
for (String aStr : strs) {
- strSet.add(aStr.trim());
+ filterList.addFilter(new TimelineExistsFilter(TimelineCompareOp.EQUAL,
+ aStr.trim()));
}
- return strSet;
+ return filterList;
}
- @SuppressWarnings("unchecked")
- private static <T> void parseKeyValues(Map<String, T> map, String str,
+ private static TimelineFilterList parseKeyValues(String str,
String pairsDelim, String keyValuesDelim, boolean stringValue,
boolean multipleValues) {
+ if (str == null) {
+ return null;
+ }
+ TimelineFilterList list = new TimelineFilterList();
String[] pairs = str.split(pairsDelim);
for (String pair : pairs) {
if (pair == null || pair.trim().isEmpty()) {
@@ -143,23 +151,28 @@ final class TimelineReaderWebServicesUtils {
try {
Object value =
GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
- map.put(pairStrs[0].trim(), (T) value);
+ list.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ pairStrs[0].trim(), value));
} catch (IOException e) {
- map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
+ list.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ pairStrs[0].trim(), pairStrs[1].trim()));
}
} else {
String key = pairStrs[0].trim();
if (multipleValues) {
- Set<String> values = new HashSet<String>();
+ Set<Object> values = new HashSet<Object>();
for (int i = 1; i < pairStrs.length; i++) {
values.add(pairStrs[i].trim());
}
- map.put(key, (T) values);
+ list.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, key, values));
} else {
- map.put(key, (T) pairStrs[1].trim());
+ list.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ key, pairStrs[1].trim()));
}
}
}
+ return list;
}
/**
@@ -175,14 +188,9 @@ final class TimelineReaderWebServicesUtils {
* @param keyValuesDelim values for a key are delimited by this delimiter.
* @return a map of key-values with each key having a set of values.
*/
- static Map<String, Set<String>> parseKeyStrValuesStr(String str,
- String pairsDelim, String keyValuesDelim) {
- if (str == null) {
- return null;
- }
- Map<String, Set<String>> map = new HashMap<String, Set<String>>();
- parseKeyValues(map, str, pairsDelim, keyValuesDelim, true, true);
- return map;
+ static TimelineFilterList parseKeyStrValuesStr(String str, String pairsDelim,
+ String keyValuesDelim) {
+ return parseKeyValues(str, pairsDelim, keyValuesDelim, true, true);
}
/**
@@ -195,14 +203,9 @@ final class TimelineReaderWebServicesUtils {
* @param keyValDelim key and value are delimited by this delimiter.
* @return a map of key-value pairs with both key and value being strings.
*/
- static Map<String, String> parseKeyStrValueStr(String str,
- String pairsDelim, String keyValDelim) {
- if (str == null) {
- return null;
- }
- Map<String, String> map = new HashMap<String, String>();
- parseKeyValues(map, str, pairsDelim, keyValDelim, true, false);
- return map;
+ static TimelineFilterList parseKeyStrValueStr(String str, String pairsDelim,
+ String keyValDelim) {
+ return parseKeyValues(str, pairsDelim, keyValDelim, true, false);
}
/**
@@ -216,14 +219,9 @@ final class TimelineReaderWebServicesUtils {
* @return a map of key-value pairs with key being a string and value, any
* object.
*/
- static Map<String, Object> parseKeyStrValueObj(String str,
- String pairsDelim, String keyValDelim) {
- if (str == null) {
- return null;
- }
- Map<String, Object> map = new HashMap<String, Object>();
- parseKeyValues(map, str, pairsDelim, keyValDelim, false, false);
- return map;
+ static TimelineFilterList parseKeyStrValueObj(String str, String pairsDelim,
+ String keyValDelim) {
+ return parseKeyValues(str, pairsDelim, keyValDelim, false, false);
}
/**
@@ -247,6 +245,20 @@ final class TimelineReaderWebServicesUtils {
return fieldList;
}
+ static TimelineFilterList parseMetricFilters(String str,
+ String delimiter) {
+ if (str == null || str.isEmpty()) {
+ return null;
+ }
+ TimelineFilterList list = new TimelineFilterList();
+ String[] strs = str.split(delimiter);
+ for (String aStr : strs) {
+ list.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, aStr.trim(), 0L));
+ }
+ return list;
+ }
+
/**
* Interpret passed string as a long.
* @param str Passed string.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
index 14e7124..81902ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
@@ -29,17 +29,27 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
@Unstable
public class TimelineCompareFilter extends TimelineFilter {
- private TimelineCompareOp compareOp;
- private String key;
- private Object value;
+ private final TimelineCompareOp compareOp;
+ private final String key;
+ private final Object value;
+ // If comparison operator is NOT_EQUAL, this flag decides if we should return
+ // the entity if key does not exist.
+ private final boolean keyMustExist;
- public TimelineCompareFilter() {
- }
-
- public TimelineCompareFilter(TimelineCompareOp op, String key, Object val) {
+ public TimelineCompareFilter(TimelineCompareOp op, String key, Object val,
+ boolean keyMustExistFlag) {
this.compareOp = op;
this.key = key;
this.value = val;
+ if (op == TimelineCompareOp.NOT_EQUAL) {
+ this.keyMustExist = keyMustExistFlag;
+ } else {
+ this.keyMustExist = true;
+ }
+ }
+
+ public TimelineCompareFilter(TimelineCompareOp op, String key, Object val) {
+ this(op, key, val, true);
}
@Override
@@ -58,4 +68,15 @@ public class TimelineCompareFilter extends TimelineFilter {
public Object getValue() {
return value;
}
+
+ public boolean getKeyMustExist() {
+ return keyMustExist;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s (%s, %s:%s:%b)",
+ this.getClass().getSimpleName(), this.compareOp.name(),
+ this.key, this.value, this.keyMustExist);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
new file mode 100644
index 0000000..36d0d7b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Filter class which represents filter to be applied based on existence of a
+ * value.
+ */
+@Private
+@Unstable
+public class TimelineExistsFilter extends TimelineFilter {
+
+ private final TimelineCompareOp compareOp;
+ private final String value;
+
+ public TimelineExistsFilter(TimelineCompareOp op, String value) {
+ this.value = value;
+ if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("CompareOp for exists filter should " +
+ "be EQUAL or NOT_EQUAL");
+ }
+ this.compareOp = op;
+ }
+
+ @Override
+ public TimelineFilterType getFilterType() {
+ return TimelineFilterType.EXISTS;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public TimelineCompareOp getCompareOp() {
+ return compareOp;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s (%s %s)",
+ this.getClass().getSimpleName(), this.compareOp.name(), this.value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilter.java
index d4b4045..5e84976 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilter.java
@@ -39,13 +39,25 @@ public abstract class TimelineFilter {
*/
LIST,
/**
- * Filter which is used for comparison.
+ * Filter which is used for key-value comparison.
*/
COMPARE,
/**
+ * Filter which is used for checking key-value equality.
+ */
+ KEY_VALUE,
+ /**
+ * Filter which is used for checking key-multiple values equality.
+ */
+ KEY_VALUES,
+ /**
* Filter which matches prefix for a config or a metric.
*/
- PREFIX
+ PREFIX,
+ /**
+ * Filter which checks existence of a value.
+ */
+ EXISTS
}
public abstract TimelineFilterType getFilterType();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
index 8727bd7..57b41a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
@@ -53,6 +53,14 @@ public class TimelineFilterList extends TimelineFilter {
this(Operator.AND, filters);
}
+ public TimelineFilterList() {
+ this(Operator.AND);
+ }
+
+ public TimelineFilterList(Operator op) {
+ this.operator = op;
+ }
+
public TimelineFilterList(Operator op, TimelineFilter...filters) {
this.operator = op;
this.filterList = new ArrayList<TimelineFilter>(Arrays.asList(filters));
@@ -88,4 +96,10 @@ public class TimelineFilterList extends TimelineFilter {
public void addFilter(TimelineFilter filter) {
filterList.add(filter);
}
+
+ @Override
+ public String toString() {
+ return String.format("TimelineFilterList %s (%d): %s",
+ this.operator, this.filterList.size(), this.filterList.toString());
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
index f902500..8cae410 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
@@ -18,25 +18,40 @@
package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+
import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
/**
* Set of utility methods used by timeline filter classes.
*/
public final class TimelineFilterUtils {
+ private static final Log LOG = LogFactory.getLog(TimelineFilterUtils.class);
+
private TimelineFilterUtils() {
}
/**
* Returns the equivalent HBase filter list's {@link Operator}.
- * @param op
+ *
+ * @param op timeline filter list operator.
* @return HBase filter list's Operator.
*/
private static Operator getHBaseOperator(TimelineFilterList.Operator op) {
@@ -52,7 +67,8 @@ public final class TimelineFilterUtils {
/**
* Returns the equivalent HBase compare filter's {@link CompareOp}.
- * @param op
+ *
+ * @param op timeline compare op.
* @return HBase compare filter's CompareOp.
*/
private static CompareOp getHBaseCompareOp(
@@ -90,6 +106,159 @@ public final class TimelineFilterUtils {
}
/**
+ * Create a HBase {@link QualifierFilter} for the passed column prefix and
+ * compare op.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param compareOp compare op.
+ * @param columnPrefix column prefix.
+ * @return a column qualifier filter.
+ */
+ public static <T> Filter createHBaseQualifierFilter(CompareOp compareOp,
+ ColumnPrefix<T> columnPrefix) {
+ return new QualifierFilter(compareOp,
+ new BinaryPrefixComparator(
+ columnPrefix.getColumnPrefixBytes("")));
+ }
+
+ /**
+ * Create filters for confs or metrics to retrieve. This list includes a
+ * configs/metrics family filter and relevant filters for confs/metrics to
+ * retrieve, if present.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param confsOrMetricToRetrieve configs/metrics to retrieve.
+ * @param columnFamily config or metric column family.
+ * @param columnPrefix config or metric column prefix.
+ * @return a filter list.
+ * @throws IOException if any problem occurs while creating the filters.
+ */
+ public static <T> Filter createFilterForConfsOrMetricsToRetrieve(
+ TimelineFilterList confsOrMetricToRetrieve, ColumnFamily<T> columnFamily,
+ ColumnPrefix<T> columnPrefix) throws IOException {
+ Filter familyFilter = new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(columnFamily.getBytes()));
+ if (confsOrMetricToRetrieve != null &&
+ !confsOrMetricToRetrieve.getFilterList().isEmpty()) {
+ // If confsOrMetricsToRetrive are specified, create a filter list based
+ // on it and family filter.
+ FilterList filter = new FilterList(familyFilter);
+ filter.addFilter(
+ createHBaseFilterList(columnPrefix, confsOrMetricToRetrieve));
+ return filter;
+ } else {
+ // Only the family filter needs to be added.
+ return familyFilter;
+ }
+ }
+
+ /**
+ * Create 2 HBase {@link SingleColumnValueFilter} filters for the specified
+ * value range represented by start and end value and wraps them inside a
+ * filter list. Start and end value should not be null.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param column Column for which single column value filter is to be created.
+ * @param startValue Start value.
+ * @param endValue End value.
+ * @return 2 single column value filters wrapped in a filter list.
+ * @throws IOException if any problem is encountered while encoding value.
+ */
+ public static <T> FilterList createSingleColValueFiltersByRange(
+ Column<T> column, Object startValue, Object endValue) throws IOException {
+ FilterList list = new FilterList();
+ Filter singleColValFilterStart = createHBaseSingleColValueFilter(
+ column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
+ column.getValueConverter().encodeValue(startValue),
+ CompareOp.GREATER_OR_EQUAL, true);
+ list.addFilter(singleColValFilterStart);
+
+ Filter singleColValFilterEnd = createHBaseSingleColValueFilter(
+ column.getColumnFamilyBytes(), column.getColumnQualifierBytes(),
+ column.getValueConverter().encodeValue(endValue),
+ CompareOp.LESS_OR_EQUAL, true);
+ list.addFilter(singleColValFilterEnd);
+ return list;
+ }
+
+ /**
+ * Creates a HBase {@link SingleColumnValueFilter}.
+ *
+ * @param columnFamily Column Family represented as bytes.
+ * @param columnQualifier Column Qualifier represented as bytes.
+ * @param value Value.
+ * @param compareOp Compare operator.
+ * @param filterIfMissing This flag decides if we should filter the row if the
+ * specified column is missing. This is based on the filter's keyMustExist
+ * field.
+ * @return a {@link SingleColumnValueFilter} object
+ * @throws IOException
+ */
+ private static SingleColumnValueFilter createHBaseSingleColValueFilter(
+ byte[] columnFamily, byte[] columnQualifier, byte[] value,
+ CompareOp compareOp, boolean filterIfMissing) throws IOException {
+ SingleColumnValueFilter singleColValFilter =
+ new SingleColumnValueFilter(columnFamily, columnQualifier, compareOp,
+ new BinaryComparator(value));
+ singleColValFilter.setLatestVersionOnly(true);
+ singleColValFilter.setFilterIfMissing(filterIfMissing);
+ return singleColValFilter;
+ }
+
+ /**
+ * Create a filter list of qualifier filters based on passed set of columns.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param colPrefix Column Prefix.
+ * @param columns set of column qualifiers.
+ * @return filter list.
+ */
+ public static <T> FilterList createFiltersFromColumnQualifiers(
+ ColumnPrefix<T> colPrefix, Set<String> columns) {
+ FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ for (String column : columns) {
+ // For columns which have compound column qualifiers (eg. events), we need
+ // to include the required separator.
+ byte[] compoundColQual =
+ colPrefix.getCompoundColQualBytes(column, (byte[])null);
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryPrefixComparator(
+ colPrefix.getColumnPrefixBytes(compoundColQual))));
+ }
+ return list;
+ }
+
+ /**
+ * Fetch columns from filter list containing exists and multivalue equality
+ * filters. This is done to fetch only required columns from back-end and
+ * then match event filters or relationships in reader.
+ *
+ * @param filterList filter list.
+ * @return set of columns.
+ */
+ public static Set<String> fetchColumnsFromFilterList(
+ TimelineFilterList filterList) {
+ Set<String> strSet = new HashSet<String>();
+ for (TimelineFilter filter : filterList.getFilterList()) {
+ switch(filter.getFilterType()) {
+ case LIST:
+ strSet.addAll(fetchColumnsFromFilterList((TimelineFilterList)filter));
+ break;
+ case KEY_VALUES:
+ strSet.add(((TimelineKeyValuesFilter)filter).getKey());
+ break;
+ case EXISTS:
+ strSet.add(((TimelineExistsFilter)filter).getValue());
+ break;
+ default:
+ LOG.info("Unexpected filter type " + filter.getFilterType());
+ break;
+ }
+ }
+ return strSet;
+ }
+
+ /**
* Creates equivalent HBase {@link FilterList} from {@link TimelineFilterList}
* while converting different timeline filters(of type {@link TimelineFilter})
* into their equivalent HBase filters.
@@ -98,22 +267,45 @@ public final class TimelineFilterUtils {
* @param colPrefix column prefix which will be used for conversion.
* @param filterList timeline filter list which has to be converted.
* @return A {@link FilterList} object.
+ * @throws IOException if any problem occurs while creating the filter list.
*/
public static <T> FilterList createHBaseFilterList(ColumnPrefix<T> colPrefix,
- TimelineFilterList filterList) {
+ TimelineFilterList filterList) throws IOException {
FilterList list =
new FilterList(getHBaseOperator(filterList.getOperator()));
for (TimelineFilter filter : filterList.getFilterList()) {
switch(filter.getFilterType()) {
case LIST:
- list.addFilter(
- createHBaseFilterList(colPrefix, (TimelineFilterList)filter));
+ list.addFilter(createHBaseFilterList(colPrefix,
+ (TimelineFilterList)filter));
break;
case PREFIX:
- list.addFilter(createHBaseColQualPrefixFilter(
- colPrefix, (TimelinePrefixFilter)filter));
+ list.addFilter(createHBaseColQualPrefixFilter(colPrefix,
+ (TimelinePrefixFilter)filter));
+ break;
+ case COMPARE:
+ TimelineCompareFilter compareFilter = (TimelineCompareFilter)filter;
+ list.addFilter(
+ createHBaseSingleColValueFilter(
+ colPrefix.getColumnFamilyBytes(),
+ colPrefix.getColumnPrefixBytes(compareFilter.getKey()),
+ colPrefix.getValueConverter().
+ encodeValue(compareFilter.getValue()),
+ getHBaseCompareOp(compareFilter.getCompareOp()),
+ compareFilter.getKeyMustExist()));
+ break;
+ case KEY_VALUE:
+ TimelineKeyValueFilter kvFilter = (TimelineKeyValueFilter)filter;
+ list.addFilter(
+ createHBaseSingleColValueFilter(
+ colPrefix.getColumnFamilyBytes(),
+ colPrefix.getColumnPrefixBytes(kvFilter.getKey()),
+ colPrefix.getValueConverter().encodeValue(kvFilter.getValue()),
+ getHBaseCompareOp(kvFilter.getCompareOp()),
+ kvFilter.getKeyMustExist()));
break;
default:
+ LOG.info("Unexpected filter type " + filter.getFilterType());
break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
new file mode 100644
index 0000000..58f0ee9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Filter class which represents filter to be applied based on key-value pair
+ * being equal or not to the values in back-end store.
+ */
+@Private
+@Unstable
+public class TimelineKeyValueFilter extends TimelineCompareFilter {
+ public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val,
+ boolean keyMustExistFlag) {
+ super(op, key, val, keyMustExistFlag);
+ if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("TimelineCompareOp for equality"
+ + " filter should be EQUAL or NOT_EQUAL");
+ }
+ }
+
+ public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val) {
+ this(op, key, val, true);
+ }
+
+ @Override
+ public TimelineFilterType getFilterType() {
+ return TimelineFilterType.KEY_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
new file mode 100644
index 0000000..0d34d47
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Filter class which represents filter to be applied based on multiple values
+ * for a key and these values being equal or not equal to values in back-end
+ * store.
+ */
+@Private
+@Unstable
+public class TimelineKeyValuesFilter extends TimelineFilter {
+ private final TimelineCompareOp compareOp;
+ private final String key;
+ private final Set<Object> values;
+ public TimelineKeyValuesFilter(TimelineCompareOp op, String key,
+ Set<Object> values) {
+ if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("TimelineCompareOp for multi value "
+ + "equality filter should be EQUAL or NOT_EQUAL");
+ }
+ this.compareOp = op;
+ this.key = key;
+ this.values = values;
+ }
+
+ @Override
+ public TimelineFilterType getFilterType() {
+ return TimelineFilterType.KEY_VALUES;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public Set<Object> getValues() {
+ return values;
+ }
+
+ public TimelineCompareOp getCompareOp() {
+ return compareOp;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s (%s, %s:%s)",
+ this.getClass().getSimpleName(), this.compareOp.name(),
+ this.key, (values == null) ? "" : values.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
index 6233f26..f36e593 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
@@ -53,4 +53,10 @@ public class TimelinePrefixFilter extends TimelineFilter {
public TimelineCompareOp getCompareOp() {
return compareOp;
}
+
+ @Override
+ public String toString() {
+ return String.format("%s (%s %s)",
+ this.getClass().getSimpleName(), this.compareOp.name(), this.prefix);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 97e05dd..bdddd7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -296,39 +296,39 @@ public class FileSystemTimelineReaderImpl extends AbstractService
continue;
}
if (filters.getRelatesTo() != null &&
- !filters.getRelatesTo().isEmpty() &&
- !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), filters.getRelatesTo())) {
+ !filters.getRelatesTo().getFilterList().isEmpty() &&
+ !TimelineStorageUtils.matchRelatesTo(entity,
+ filters.getRelatesTo())) {
continue;
}
if (filters.getIsRelatedTo() != null &&
- !filters.getIsRelatedTo().isEmpty() &&
- !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) {
+ !filters.getIsRelatedTo().getFilterList().isEmpty() &&
+ !TimelineStorageUtils.matchIsRelatedTo(entity,
+ filters.getIsRelatedTo())) {
continue;
}
if (filters.getInfoFilters() != null &&
- !filters.getInfoFilters().isEmpty() &&
- !TimelineStorageUtils.matchFilters(
- entity.getInfo(), filters.getInfoFilters())) {
+ !filters.getInfoFilters().getFilterList().isEmpty() &&
+ !TimelineStorageUtils.matchInfoFilters(entity,
+ filters.getInfoFilters())) {
continue;
}
if (filters.getConfigFilters() != null &&
- !filters.getConfigFilters().isEmpty() &&
- !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), filters.getConfigFilters())) {
+ !filters.getConfigFilters().getFilterList().isEmpty() &&
+ !TimelineStorageUtils.matchConfigFilters(entity,
+ filters.getConfigFilters())) {
continue;
}
if (filters.getMetricFilters() != null &&
- !filters.getMetricFilters().isEmpty() &&
- !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), filters.getMetricFilters())) {
+ !filters.getMetricFilters().getFilterList().isEmpty() &&
+ !TimelineStorageUtils.matchMetricFilters(entity,
+ filters.getMetricFilters())) {
continue;
}
if (filters.getEventFilters() != null &&
- !filters.getEventFilters().isEmpty() &&
- !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), filters.getEventFilters())) {
+ !filters.getEventFilters().getFilterList().isEmpty() &&
+ !TimelineStorageUtils.matchEventFilters(entity,
+ filters.getEventFilters())) {
continue;
}
TimelineEntity entityToBeReturned = createEntityToBeReturned(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index b75007d..172f982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -407,36 +407,39 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
"! Using the current timestamp");
eventTimestamp = System.currentTimeMillis();
}
- byte[] columnQualifierFirst =
- Bytes.toBytes(Separator.VALUES.encode(eventId));
- byte[] columnQualifierWithTsBytes = Separator.VALUES.
- join(columnQualifierFirst, Bytes.toBytes(
- TimelineStorageUtils.invertLong(eventTimestamp)));
+ byte[] eventTs =
+ Bytes.toBytes(TimelineStorageUtils.invertLong(eventTimestamp));
Map<String, Object> eventInfo = event.getInfo();
if ((eventInfo == null) || (eventInfo.size() == 0)) {
- // add separator since event key is empty
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(columnQualifierWithTsBytes,
- null);
if (isApplication) {
+ byte[] compoundColumnQualifierBytes =
+ ApplicationColumnPrefix.EVENT.
+ getCompoundColQualBytes(eventId, eventTs, null);
ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
compoundColumnQualifierBytes, null,
- TimelineStorageUtils.EMPTY_BYTES);
+ TimelineStorageUtils.EMPTY_BYTES);
} else {
+ byte[] compoundColumnQualifierBytes =
+ EntityColumnPrefix.EVENT.
+ getCompoundColQualBytes(eventId, eventTs, null);
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
compoundColumnQualifierBytes, null,
- TimelineStorageUtils.EMPTY_BYTES);
+ TimelineStorageUtils.EMPTY_BYTES);
}
} else {
for (Map.Entry<String, Object> info : eventInfo.entrySet()) {
// eventId?infoKey
- byte[] compoundColumnQualifierBytes =
- Separator.VALUES.join(columnQualifierWithTsBytes,
- Bytes.toBytes(info.getKey()));
+ byte[] infoKey = Bytes.toBytes(info.getKey());
if (isApplication) {
+ byte[] compoundColumnQualifierBytes =
+ ApplicationColumnPrefix.EVENT.
+ getCompoundColQualBytes(eventId, eventTs, infoKey);
ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
compoundColumnQualifierBytes, null, info.getValue());
} else {
+ byte[] compoundColumnQualifierBytes =
+ EntityColumnPrefix.EVENT.
+ getCompoundColQualBytes(eventId, eventTs, infoKey);
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
compoundColumnQualifierBytes, null, info.getValue());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
index 5734389..80fcf8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -24,8 +24,11 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
@@ -41,7 +44,8 @@ public enum ApplicationColumn implements Column<ApplicationTable> {
/**
* When the application was created.
*/
- CREATED_TIME(ApplicationColumnFamily.INFO, "created_time"),
+ CREATED_TIME(ApplicationColumnFamily.INFO, "created_time",
+ LongConverter.getInstance()),
/**
* The version of the flow that this app belongs to.
@@ -55,12 +59,17 @@ public enum ApplicationColumn implements Column<ApplicationTable> {
private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
String columnQualifier) {
+ this(columnFamily, columnQualifier, GenericConverter.getInstance());
+ }
+
+ private ApplicationColumn(ColumnFamily<ApplicationTable> columnFamily,
+ String columnQualifier, ValueConverter converter) {
this.columnFamily = columnFamily;
this.columnQualifier = columnQualifier;
// Future-proof by ensuring the right column prefix hygiene.
this.columnQualifierBytes =
Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
- this.column = new ColumnHelper<ApplicationTable>(columnFamily);
+ this.column = new ColumnHelper<ApplicationTable>(columnFamily, converter);
}
/**
@@ -81,6 +90,21 @@ public enum ApplicationColumn implements Column<ApplicationTable> {
return column.readResult(result, columnQualifierBytes);
}
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return column.getValueConverter();
+ }
+
/**
* Retrieve an {@link ApplicationColumn} given a name, or null if there is no
* match. The following holds true: {@code columnFor(x) == columnFor(y)} if
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
index 9120f3d..1dfc4db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -56,7 +56,7 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
/**
* Lifecycle events for an application.
*/
- EVENT(ApplicationColumnFamily.INFO, "e"),
+ EVENT(ApplicationColumnFamily.INFO, "e", true),
/**
* Config column stores configuration with config key as the column name.
@@ -78,6 +78,7 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
*/
private final String columnPrefix;
private final byte[] columnPrefixBytes;
+ private final boolean compoundColQual;
/**
* Private constructor, meant to be used by the enum definition.
@@ -87,7 +88,18 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
*/
private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
String columnPrefix) {
- this(columnFamily, columnPrefix, GenericConverter.getInstance());
+ this(columnFamily, columnPrefix, false, GenericConverter.getInstance());
+ }
+
+ private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
+ String columnPrefix, boolean compoundColQual) {
+ this(columnFamily, columnPrefix, compoundColQual,
+ GenericConverter.getInstance());
+ }
+
+ private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
+ String columnPrefix, ValueConverter converter) {
+ this(columnFamily, columnPrefix, false, converter);
}
/**
@@ -99,7 +111,7 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
* this column prefix.
*/
private ApplicationColumnPrefix(ColumnFamily<ApplicationTable> columnFamily,
- String columnPrefix, ValueConverter converter) {
+ String columnPrefix, boolean compoundColQual, ValueConverter converter) {
column = new ColumnHelper<ApplicationTable>(columnFamily, converter);
this.columnFamily = columnFamily;
this.columnPrefix = columnPrefix;
@@ -110,6 +122,7 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
this.columnPrefixBytes =
Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
}
+ this.compoundColQual = compoundColQual;
}
/**
@@ -131,6 +144,20 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
this.columnPrefixBytes, qualifierPrefix);
}
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public byte[] getCompoundColQualBytes(String qualifier,
+ byte[]...components) {
+ if (!compoundColQual) {
+ return ColumnHelper.getColumnQualifier(null, qualifier);
+ }
+ return ColumnHelper.getCompoundColumnQualifierBytes(qualifier, components);
+ }
+
/*
* (non-Javadoc)
*
@@ -196,6 +223,10 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
return column.readResult(result, columnQualifier);
}
+ public ValueConverter getValueConverter() {
+ return column.getValueConverter();
+ }
+
/*
* (non-Javadoc)
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
index a8e1c66..ff61633 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
import java.io.IOException;
@@ -71,6 +72,11 @@ public enum AppToFlowColumn implements Column<AppToFlowTable> {
return columnQualifier;
}
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
public void store(byte[] rowKey,
TypedBufferedMutator<AppToFlowTable> tableMutator, Long timestamp,
Object inputValue, Attribute... attributes) throws IOException {
@@ -78,6 +84,16 @@ public enum AppToFlowColumn implements Column<AppToFlowTable> {
inputValue, attributes);
}
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return column.getValueConverter();
+ }
+
public Object readResult(Result result) throws IOException {
return column.readResult(result, columnQualifierBytes);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
index 1f0b48f..90f2de4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
@@ -60,4 +60,21 @@ public interface Column<T> {
*/
Object readResult(Result result) throws IOException;
+ /**
+ * Returns column family name(as bytes) associated with this column.
+ * @return a byte array encoding column family for this column qualifier.
+ */
+ byte[] getColumnFamilyBytes();
+
+ /**
+ * Get byte representation for this column qualifier.
+ * @return a byte array representing column qualifier.
+ */
+ byte[] getColumnQualifierBytes();
+
+ /**
+ * Returns value converter implementation associated with this column.
+ * @return a {@link ValueConverter} implementation.
+ */
+ ValueConverter getValueConverter();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index 15bb818..4adb413 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -363,6 +363,22 @@ public class ColumnHelper<T> {
}
/**
+ * Create a compound column qualifier by combining qualifier and components.
+ *
+ * @param qualifier Column QUalifier.
+ * @param components Other components.
+ * @return a byte array representing compound column qualifier.
+ */
+ public static byte[] getCompoundColumnQualifierBytes(String qualifier,
+ byte[]...components) {
+ byte[] colQualBytes = Bytes.toBytes(Separator.VALUES.encode(qualifier));
+ for (int i = 0; i < components.length; i++) {
+ colQualBytes = Separator.VALUES.join(colQualBytes, components[i]);
+ }
+ return colQualBytes;
+ }
+
+ /**
* @param columnPrefixBytes The byte representation for the column prefix.
* Should not contain {@link Separator#QUALIFIERS}.
* @param qualifier for the remainder of the column.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index f221b31..e4b7f16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -112,6 +112,18 @@ public interface ColumnPrefix<T> {
readResultsWithTimestamps(Result result) throws IOException;
/**
+ * @param result from which to read columns
+ * @return the latest values of columns in the column family. The column
+ * qualifier is returned as a list of parts, each part a byte[]. This
+ * is to facilitate returning byte arrays of values that were not
+ * Strings. If they can be treated as Strings, you should use
+ * {@link #readResults(Result)} instead.
+ * @throws IOException if any problem occurs while reading results.
+ */
+ Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException;
+
+ /**
* @param qualifierPrefix Column qualifier or prefix of qualifier.
* @return a byte array encoding column prefix and qualifier/prefix passed.
*/
@@ -122,4 +134,27 @@ public interface ColumnPrefix<T> {
* @return a byte array encoding column prefix and qualifier/prefix passed.
*/
byte[] getColumnPrefixBytes(byte[] qualifierPrefix);
+
+ /**
+ * Returns column family name(as bytes) associated with this column prefix.
+ * @return a byte array encoding column family for this prefix.
+ */
+ byte[] getColumnFamilyBytes();
+
+ /**
+ * Returns value converter implementation associated with this column prefix.
+ * @return a {@link ValueConverter} implementation.
+ */
+ ValueConverter getValueConverter();
+
+ /**
+ * Get compound column qualifier bytes if the column qualifier is a compound
+ * qualifier. Returns the qualifier passed as bytes if the column is not a
+ * compound column qualifier.
+ *
+ * @param qualifier Column Qualifier.
+ * @param components Other components.
+ * @return byte array representing compound column qualifier.
+ */
+ byte[] getCompoundColQualBytes(String qualifier, byte[]...components);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
new file mode 100644
index 0000000..4099e92
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter.TimelineFilterType;
+
+/**
+ * Used to define which filter to match.
+ */
+enum TimelineEntityFiltersType {
+ CONFIG {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUE;
+ }
+ },
+ INFO {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUE;
+ }
+ },
+ METRIC {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.COMPARE;
+ }
+ },
+ EVENT {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.EXISTS;
+ }
+ },
+ IS_RELATED_TO {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUES;
+ }
+ },
+ RELATES_TO {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUES;
+ }
+ };
+
+ /**
+ * Checks whether filter type is valid for the filter being matched.
+ *
+ * @param filterType filter type.
+ * @return true, if its a valid filter, false otherwise.
+ */
+ abstract boolean isValidFilter(TimelineFilterType filterType);
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: Addendum to YARN-3863. Deleted
files that were added incorrectly.
Posted by gt...@apache.org.
Addendum to YARN-3863. Deleted files that were added incorrectly.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/974a9d7c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/974a9d7c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/974a9d7c
Branch: refs/heads/YARN-2928
Commit: 974a9d7cad7162c87576a92d28774592114d1317
Parents: 2b2df86
Author: Sangjin Lee <sj...@apache.org>
Authored: Tue Apr 12 12:32:43 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:33 2016 -0700
----------------------------------------------------------------------
.../reader/filter/TimelineExistsFilter.java | 62 -----------------
.../reader/filter/TimelineKeyValueFilter.java | 48 -------------
.../reader/filter/TimelineKeyValuesFilter.java | 71 --------------------
.../common/TimelineEntityFiltersType.java | 71 --------------------
4 files changed, 252 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/974a9d7c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
deleted file mode 100644
index 36d0d7b..0000000
--- a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * Filter class which represents filter to be applied based on existence of a
- * value.
- */
-@Private
-@Unstable
-public class TimelineExistsFilter extends TimelineFilter {
-
- private final TimelineCompareOp compareOp;
- private final String value;
-
- public TimelineExistsFilter(TimelineCompareOp op, String value) {
- this.value = value;
- if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
- throw new IllegalArgumentException("CompareOp for exists filter should " +
- "be EQUAL or NOT_EQUAL");
- }
- this.compareOp = op;
- }
-
- @Override
- public TimelineFilterType getFilterType() {
- return TimelineFilterType.EXISTS;
- }
-
- public String getValue() {
- return value;
- }
-
- public TimelineCompareOp getCompareOp() {
- return compareOp;
- }
-
- @Override
- public String toString() {
- return String.format("%s (%s %s)",
- this.getClass().getSimpleName(), this.compareOp.name(), this.value);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/974a9d7c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
deleted file mode 100644
index 58f0ee9..0000000
--- a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * Filter class which represents filter to be applied based on key-value pair
- * being equal or not to the values in back-end store.
- */
-@Private
-@Unstable
-public class TimelineKeyValueFilter extends TimelineCompareFilter {
- public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val,
- boolean keyMustExistFlag) {
- super(op, key, val, keyMustExistFlag);
- if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
- throw new IllegalArgumentException("TimelineCompareOp for equality"
- + " filter should be EQUAL or NOT_EQUAL");
- }
- }
-
- public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val) {
- this(op, key, val, true);
- }
-
- @Override
- public TimelineFilterType getFilterType() {
- return TimelineFilterType.KEY_VALUE;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/974a9d7c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
deleted file mode 100644
index 0d34d47..0000000
--- a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
-
-import java.util.Set;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * Filter class which represents filter to be applied based on multiple values
- * for a key and these values being equal or not equal to values in back-end
- * store.
- */
-@Private
-@Unstable
-public class TimelineKeyValuesFilter extends TimelineFilter {
- private final TimelineCompareOp compareOp;
- private final String key;
- private final Set<Object> values;
- public TimelineKeyValuesFilter(TimelineCompareOp op, String key,
- Set<Object> values) {
- if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
- throw new IllegalArgumentException("TimelineCompareOp for multi value "
- + "equality filter should be EQUAL or NOT_EQUAL");
- }
- this.compareOp = op;
- this.key = key;
- this.values = values;
- }
-
- @Override
- public TimelineFilterType getFilterType() {
- return TimelineFilterType.KEY_VALUES;
- }
-
- public String getKey() {
- return key;
- }
-
- public Set<Object> getValues() {
- return values;
- }
-
- public TimelineCompareOp getCompareOp() {
- return compareOp;
- }
-
- @Override
- public String toString() {
- return String.format("%s (%s, %s:%s)",
- this.getClass().getSimpleName(), this.compareOp.name(),
- this.key, (values == null) ? "" : values.toString());
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/974a9d7c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
deleted file mode 100644
index 4099e92..0000000
--- a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.timelineservice.storage.common;
-
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter.TimelineFilterType;
-
-/**
- * Used to define which filter to match.
- */
-enum TimelineEntityFiltersType {
- CONFIG {
- boolean isValidFilter(TimelineFilterType filterType) {
- return filterType == TimelineFilterType.LIST ||
- filterType == TimelineFilterType.KEY_VALUE;
- }
- },
- INFO {
- boolean isValidFilter(TimelineFilterType filterType) {
- return filterType == TimelineFilterType.LIST ||
- filterType == TimelineFilterType.KEY_VALUE;
- }
- },
- METRIC {
- boolean isValidFilter(TimelineFilterType filterType) {
- return filterType == TimelineFilterType.LIST ||
- filterType == TimelineFilterType.COMPARE;
- }
- },
- EVENT {
- boolean isValidFilter(TimelineFilterType filterType) {
- return filterType == TimelineFilterType.LIST ||
- filterType == TimelineFilterType.EXISTS;
- }
- },
- IS_RELATED_TO {
- boolean isValidFilter(TimelineFilterType filterType) {
- return filterType == TimelineFilterType.LIST ||
- filterType == TimelineFilterType.KEY_VALUES;
- }
- },
- RELATES_TO {
- boolean isValidFilter(TimelineFilterType filterType) {
- return filterType == TimelineFilterType.LIST ||
- filterType == TimelineFilterType.KEY_VALUES;
- }
- };
-
- /**
- * Checks whether filter type is valid for the filter being matched.
- *
- * @param filterType filter type.
- * @return true, if its a valid filter, false otherwise.
- */
- abstract boolean isValidFilter(TimelineFilterType filterType);
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: YARN-4200. Refactor reader classes
in storage to nest under hbase specific package name. Contributed by Li Lu.
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
new file mode 100644
index 0000000..181ec81
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -0,0 +1,383 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for application entities that are stored in the
+ * application table.
+ */
+class ApplicationEntityReader extends GenericEntityReader {
+ private static final ApplicationTable APPLICATION_TABLE =
+ new ApplicationTable();
+
+ public ApplicationEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
+ true);
+ }
+
+ public ApplicationEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ String entityId, TimelineFilterList confsToRetrieve,
+ TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
+ confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link ApplicationTable}.
+ */
+ protected BaseTable<?> getTable() {
+ return APPLICATION_TABLE;
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() {
+ FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ // Fetch all the columns.
+ if (fieldsToRetrieve.contains(Field.ALL) &&
+ (confsToRetrieve == null ||
+ confsToRetrieve.getFilterList().isEmpty()) &&
+ (metricsToRetrieve == null ||
+ metricsToRetrieve.getFilterList().isEmpty())) {
+ return list;
+ }
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ // Events not required.
+ if (!fieldsToRetrieve.contains(Field.EVENTS) &&
+ !fieldsToRetrieve.contains(Field.ALL) && eventFilters == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ ApplicationColumnPrefix.EVENT.getColumnPrefixBytes(""))));
+ }
+ // info not required.
+ if (!fieldsToRetrieve.contains(Field.INFO) &&
+ !fieldsToRetrieve.contains(Field.ALL) && infoFilters == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ ApplicationColumnPrefix.INFO.getColumnPrefixBytes(""))));
+ }
+ // is releated to not required.
+ if (!fieldsToRetrieve.contains(Field.IS_RELATED_TO) &&
+ !fieldsToRetrieve.contains(Field.ALL) && isRelatedTo == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ ApplicationColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
+ }
+ // relates to not required.
+ if (!fieldsToRetrieve.contains(Field.RELATES_TO) &&
+ !fieldsToRetrieve.contains(Field.ALL) && relatesTo == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ ApplicationColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
+ }
+ list.addFilter(infoColFamilyList);
+ if ((fieldsToRetrieve.contains(Field.CONFIGS) || configFilters != null) ||
+ (confsToRetrieve != null &&
+ !confsToRetrieve.getFilterList().isEmpty())) {
+ FilterList filterCfg =
+ new FilterList(new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(ApplicationColumnFamily.CONFIGS.getBytes())));
+ if (confsToRetrieve != null &&
+ !confsToRetrieve.getFilterList().isEmpty()) {
+ filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.CONFIG, confsToRetrieve));
+ }
+ list.addFilter(filterCfg);
+ }
+ if ((fieldsToRetrieve.contains(Field.METRICS) || metricFilters != null) ||
+ (metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty())) {
+ FilterList filterMetrics =
+ new FilterList(new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(ApplicationColumnFamily.METRICS.getBytes())));
+ if (metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.METRIC, metricsToRetrieve));
+ }
+ list.addFilter(filterMetrics);
+ }
+ return list;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ byte[] rowKey =
+ ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
+ appId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ get.setFilter(filterList);
+ }
+ return table.getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ if (singleEntityRead) {
+ Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ } else {
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ if (singleEntityRead) {
+ if (flowName == null || flowRunId == null || userId == null) {
+ FlowContext context =
+ lookupFlowContext(clusterId, appId, hbaseConf, conn);
+ flowName = context.flowName;
+ flowRunId = context.flowRunId;
+ userId = context.userId;
+ }
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+ if (!fieldsToRetrieve.contains(Field.CONFIGS) &&
+ confsToRetrieve != null && !confsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.CONFIGS);
+ }
+ if (!fieldsToRetrieve.contains(Field.METRICS) &&
+ metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.METRICS);
+ }
+ if (!singleEntityRead) {
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (modifiedTimeBegin == null) {
+ modifiedTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (modifiedTimeEnd == null) {
+ modifiedTimeEnd = DEFAULT_END_TIME;
+ }
+ }
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException {
+ Scan scan = new Scan();
+ if (flowRunId != null) {
+ scan.setRowPrefixFilter(ApplicationRowKey.
+ getRowKeyPrefix(clusterId, userId, flowName, flowRunId));
+ } else {
+ scan.setRowPrefixFilter(ApplicationRowKey.
+ getRowKeyPrefix(clusterId, userId, flowName));
+ }
+ FilterList newList = new FilterList();
+ newList.addFilter(new PageFilter(limit));
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ newList.addFilter(filterList);
+ }
+ scan.setFilter(newList);
+ return table.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
+ String entityId = ApplicationColumn.ID.readResult(result).toString();
+ entity.setId(entityId);
+
+ // fetch created time
+ Number createdTime =
+ (Number)ApplicationColumn.CREATED_TIME.readResult(result);
+ entity.setCreatedTime(createdTime.longValue());
+ if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
+ entity.getCreatedTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // fetch modified time
+ Number modifiedTime =
+ (Number)ApplicationColumn.MODIFIED_TIME.readResult(result);
+ entity.setModifiedTime(modifiedTime.longValue());
+ if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
+ entity.getModifiedTime() > modifiedTimeEnd)) {
+ return null;
+ }
+
+ // fetch is related to entities
+ boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
+ true);
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
+ entity.getIsRelatedToEntities(), isRelatedTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities
+ boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
+ false);
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
+ entity.getRelatesToEntities(), relatesTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info
+ boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
+ if (checkInfo &&
+ !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.INFO)) {
+ entity.getInfo().clear();
+ }
+ }
+
+ // fetch configs
+ boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
+ readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
+ if (checkConfigs && !TimelineStorageUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.CONFIGS)) {
+ entity.getConfigs().clear();
+ }
+ }
+
+ // fetch events
+ boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, true);
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics
+ boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
+ if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.METRICS)) {
+ entity.getMetrics().clear();
+ }
+ }
+ return entity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
new file mode 100644
index 0000000..52ceef8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for flow activity entities that are stored in the
+ * flow activity table.
+ */
+class FlowActivityEntityReader extends TimelineEntityReader {
+ private static final FlowActivityTable FLOW_ACTIVITY_TABLE =
+ new FlowActivityTable();
+
+ public FlowActivityEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, null, null, fieldsToRetrieve, true);
+ }
+
+ public FlowActivityEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ String entityId, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
+ null, null, fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link FlowActivityTable}.
+ */
+ @Override
+ protected BaseTable<?> getTable() {
+ return FLOW_ACTIVITY_TABLE;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() {
+ return null;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ throw new UnsupportedOperationException(
+ "we don't support a single entity query");
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException {
+ Scan scan = new Scan();
+ if (createdTimeBegin == DEFAULT_BEGIN_TIME &&
+ createdTimeEnd == DEFAULT_END_TIME) {
+ scan.setRowPrefixFilter(FlowActivityRowKey.getRowKeyPrefix(clusterId));
+ } else {
+ scan.setStartRow(
+ FlowActivityRowKey.getRowKeyPrefix(clusterId, createdTimeEnd));
+ scan.setStopRow(
+ FlowActivityRowKey.getRowKeyPrefix(clusterId,
+ (createdTimeBegin <= 0 ? 0: (createdTimeBegin - 1))));
+ }
+ // use the page filter to limit the result to the page size
+ // the scanner may still return more than the limit; therefore we need to
+ // read the right number as we iterate
+ scan.setFilter(new PageFilter(limit));
+ return table.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(result.getRow());
+
+ long time = rowKey.getDayTimestamp();
+ String user = rowKey.getUserId();
+ String flowName = rowKey.getFlowName();
+
+ FlowActivityEntity flowActivity =
+ new FlowActivityEntity(clusterId, time, user, flowName);
+ // set the id
+ flowActivity.setId(flowActivity.getId());
+ // get the list of run ids along with the version that are associated with
+ // this flow on this day
+ Map<String, Object> runIdsMap =
+ FlowActivityColumnPrefix.RUN_ID.readResults(result);
+ for (Map.Entry<String, Object> e : runIdsMap.entrySet()) {
+ Long runId = Long.valueOf(e.getKey());
+ String version = (String)e.getValue();
+ FlowRunEntity flowRun = new FlowRunEntity();
+ flowRun.setUser(user);
+ flowRun.setName(flowName);
+ flowRun.setRunId(runId);
+ flowRun.setVersion(version);
+ // set the id
+ flowRun.setId(flowRun.getId());
+ flowActivity.addFlowRun(flowRun);
+ }
+
+ return flowActivity;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
new file mode 100644
index 0000000..6286ee1
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -0,0 +1,226 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.PageFilter;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for flow run entities that are stored in the flow run
+ * table.
+ */
+class FlowRunEntityReader extends TimelineEntityReader {
+ private static final FlowRunTable FLOW_RUN_TABLE = new FlowRunTable();
+
+ public FlowRunEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
+ EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, null, metricsToRetrieve, fieldsToRetrieve, true);
+ }
+
+ public FlowRunEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ String entityId, TimelineFilterList confsToRetrieve,
+ TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
+ null, metricsToRetrieve, fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link FlowRunTable}.
+ */
+ @Override
+ protected BaseTable<?> getTable() {
+ return FLOW_RUN_TABLE;
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(userId, "userId shouldn't be null");
+ Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
+ if (singleEntityRead) {
+ Preconditions.checkNotNull(flowRunId, "flowRunId shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn) {
+ if (!singleEntityRead) {
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (!fieldsToRetrieve.contains(Field.METRICS) &&
+ metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.METRICS);
+ }
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() {
+ FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(FlowRunColumnFamily.INFO.getBytes()));
+ // Metrics not required.
+ if (!singleEntityRead && !fieldsToRetrieve.contains(Field.METRICS) &&
+ !fieldsToRetrieve.contains(Field.ALL)) {
+ FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
+ infoColFamilyList.addFilter(infoColumnFamily);
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(""))));
+ list.addFilter(infoColFamilyList);
+ }
+ if (metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ FilterList infoColFamilyList = new FilterList();
+ infoColFamilyList.addFilter(infoColumnFamily);
+ infoColFamilyList.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ FlowRunColumnPrefix.METRIC, metricsToRetrieve));
+ list.addFilter(infoColFamilyList);
+ }
+ return list;
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ byte[] rowKey =
+ FlowRunRowKey.getRowKey(clusterId, userId, flowName, flowRunId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ get.setFilter(filterList);
+ }
+ return table.getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException {
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(
+ FlowRunRowKey.getRowKeyPrefix(clusterId, userId, flowName));
+ FilterList newList = new FilterList();
+ newList.addFilter(new PageFilter(limit));
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ newList.addFilter(filterList);
+ }
+ scan.setFilter(newList);
+ return table.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ FlowRunEntity flowRun = new FlowRunEntity();
+ flowRun.setUser(userId);
+ flowRun.setName(flowName);
+ if (singleEntityRead) {
+ flowRun.setRunId(flowRunId);
+ } else {
+ FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
+ flowRun.setRunId(rowKey.getFlowRunId());
+ }
+
+ // read the start time
+ Long startTime = (Long)FlowRunColumn.MIN_START_TIME.readResult(result);
+ if (startTime != null) {
+ flowRun.setStartTime(startTime.longValue());
+ }
+ if (!singleEntityRead && (flowRun.getStartTime() < createdTimeBegin ||
+ flowRun.getStartTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // read the end time if available
+ Long endTime = (Long)FlowRunColumn.MAX_END_TIME.readResult(result);
+ if (endTime != null) {
+ flowRun.setMaxEndTime(endTime.longValue());
+ }
+
+ // read the flow version
+ String version = (String)FlowRunColumn.FLOW_VERSION.readResult(result);
+ if (version != null) {
+ flowRun.setVersion(version);
+ }
+
+ // read metrics
+ if (singleEntityRead || fieldsToRetrieve.contains(Field.METRICS)) {
+ readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
+ }
+
+ // set the id
+ flowRun.setId(flowRun.getId());
+ return flowRun;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
new file mode 100644
index 0000000..f3f380c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -0,0 +1,497 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
+import org.apache.hadoop.hbase.filter.FamilyFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
+import org.apache.hadoop.hbase.filter.FilterList.Operator;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * Timeline entity reader for generic entities that are stored in the entity
+ * table.
+ */
+class GenericEntityReader extends TimelineEntityReader {
+ private static final EntityTable ENTITY_TABLE = new EntityTable();
+ private static final Log LOG = LogFactory.getLog(GenericEntityReader.class);
+
+ /**
+ * Used to look up the flow context.
+ */
+ private final AppToFlowTable appToFlowTable = new AppToFlowTable();
+
+ public GenericEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
+ EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
+ createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
+ relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
+ eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
+ sortedKeys);
+ }
+
+ public GenericEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ String entityId, TimelineFilterList confsToRetrieve,
+ TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
+ super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
+ confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
+ }
+
+ /**
+ * Uses the {@link EntityTable}.
+ */
+ protected BaseTable<?> getTable() {
+ return ENTITY_TABLE;
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() {
+ FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ // Fetch all the columns.
+ if (fieldsToRetrieve.contains(Field.ALL) &&
+ (confsToRetrieve == null ||
+ confsToRetrieve.getFilterList().isEmpty()) &&
+ (metricsToRetrieve == null ||
+ metricsToRetrieve.getFilterList().isEmpty())) {
+ return list;
+ }
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(EntityColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ // Events not required.
+ if (!fieldsToRetrieve.contains(Field.EVENTS) &&
+ !fieldsToRetrieve.contains(Field.ALL) && eventFilters == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ EntityColumnPrefix.EVENT.getColumnPrefixBytes(""))));
+ }
+ // info not required.
+ if (!fieldsToRetrieve.contains(Field.INFO) &&
+ !fieldsToRetrieve.contains(Field.ALL) && infoFilters == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ EntityColumnPrefix.INFO.getColumnPrefixBytes(""))));
+ }
+ // is related to not required.
+ if (!fieldsToRetrieve.contains(Field.IS_RELATED_TO) &&
+ !fieldsToRetrieve.contains(Field.ALL) && isRelatedTo == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ EntityColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
+ }
+ // relates to not required.
+ if (!fieldsToRetrieve.contains(Field.RELATES_TO) &&
+ !fieldsToRetrieve.contains(Field.ALL) && relatesTo == null) {
+ infoColFamilyList.addFilter(
+ new QualifierFilter(CompareOp.NOT_EQUAL,
+ new BinaryPrefixComparator(
+ EntityColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
+ }
+ list.addFilter(infoColFamilyList);
+ if ((fieldsToRetrieve.contains(Field.CONFIGS) || configFilters != null) ||
+ (confsToRetrieve != null &&
+ !confsToRetrieve.getFilterList().isEmpty())) {
+ FilterList filterCfg =
+ new FilterList(new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(EntityColumnFamily.CONFIGS.getBytes())));
+ if (confsToRetrieve != null &&
+ !confsToRetrieve.getFilterList().isEmpty()) {
+ filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.CONFIG, confsToRetrieve));
+ }
+ list.addFilter(filterCfg);
+ }
+ if ((fieldsToRetrieve.contains(Field.METRICS) || metricFilters != null) ||
+ (metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty())) {
+ FilterList filterMetrics =
+ new FilterList(new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(EntityColumnFamily.METRICS.getBytes())));
+ if (metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.METRIC, metricsToRetrieve));
+ }
+ list.addFilter(filterMetrics);
+ }
+ return list;
+ }
+
+ protected FlowContext lookupFlowContext(String clusterId, String appId,
+ Configuration hbaseConf, Connection conn) throws IOException {
+ byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
+ Get get = new Get(rowKey);
+ Result result = appToFlowTable.getResult(hbaseConf, conn, get);
+ if (result != null && !result.isEmpty()) {
+ return new FlowContext(
+ AppToFlowColumn.USER_ID.readResult(result).toString(),
+ AppToFlowColumn.FLOW_ID.readResult(result).toString(),
+ ((Number)AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
+ } else {
+ throw new IOException(
+ "Unable to find the context flow ID and flow run ID for clusterId=" +
+ clusterId + ", appId=" + appId);
+ }
+ }
+
+ protected static class FlowContext {
+ protected final String userId;
+ protected final String flowName;
+ protected final Long flowRunId;
+ public FlowContext(String user, String flowName, Long flowRunId) {
+ this.userId = user;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ }
+ }
+
+ @Override
+ protected void validateParams() {
+ Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ if (singleEntityRead) {
+ Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
+ }
+ }
+
+ @Override
+ protected void augmentParams(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ // In reality all three should be null or neither should be null
+ if (flowName == null || flowRunId == null || userId == null) {
+ FlowContext context =
+ lookupFlowContext(clusterId, appId, hbaseConf, conn);
+ flowName = context.flowName;
+ flowRunId = context.flowRunId;
+ userId = context.userId;
+ }
+ if (fieldsToRetrieve == null) {
+ fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+ if (!fieldsToRetrieve.contains(Field.CONFIGS) &&
+ confsToRetrieve != null && !confsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.CONFIGS);
+ }
+ if (!fieldsToRetrieve.contains(Field.METRICS) &&
+ metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.METRICS);
+ }
+ if (!singleEntityRead) {
+ if (limit == null || limit < 0) {
+ limit = TimelineReader.DEFAULT_LIMIT;
+ }
+ if (createdTimeBegin == null) {
+ createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (createdTimeEnd == null) {
+ createdTimeEnd = DEFAULT_END_TIME;
+ }
+ if (modifiedTimeBegin == null) {
+ modifiedTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ if (modifiedTimeEnd == null) {
+ modifiedTimeEnd = DEFAULT_END_TIME;
+ }
+ }
+ }
+
+ @Override
+ protected Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException {
+ byte[] rowKey =
+ EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
+ entityType, entityId);
+ Get get = new Get(rowKey);
+ get.setMaxVersions(Integer.MAX_VALUE);
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ get.setFilter(filterList);
+ }
+ return table.getResult(hbaseConf, conn, get);
+ }
+
+ @Override
+ protected ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException {
+ // Scan through part of the table to find the entities belong to one app
+ // and one type
+ Scan scan = new Scan();
+ scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
+ clusterId, userId, flowName, flowRunId, appId, entityType));
+ scan.setMaxVersions(Integer.MAX_VALUE);
+ if (filterList != null && !filterList.getFilters().isEmpty()) {
+ scan.setFilter(filterList);
+ }
+ return table.getResultScanner(hbaseConf, conn, scan);
+ }
+
+ @Override
+ protected TimelineEntity parseEntity(Result result) throws IOException {
+ if (result == null || result.isEmpty()) {
+ return null;
+ }
+ TimelineEntity entity = new TimelineEntity();
+ String entityType = EntityColumn.TYPE.readResult(result).toString();
+ entity.setType(entityType);
+ String entityId = EntityColumn.ID.readResult(result).toString();
+ entity.setId(entityId);
+
+ // fetch created time
+ Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result);
+ entity.setCreatedTime(createdTime.longValue());
+ if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
+ entity.getCreatedTime() > createdTimeEnd)) {
+ return null;
+ }
+
+ // fetch modified time
+ Number modifiedTime = (Number)EntityColumn.MODIFIED_TIME.readResult(result);
+ entity.setModifiedTime(modifiedTime.longValue());
+ if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
+ entity.getModifiedTime() > modifiedTimeEnd)) {
+ return null;
+ }
+
+ // fetch is related to entities
+ boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
+ readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
+ entity.getIsRelatedToEntities(), isRelatedTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ entity.getIsRelatedToEntities().clear();
+ }
+ }
+
+ // fetch relates to entities
+ boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
+ readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
+ entity.getRelatesToEntities(), relatesTo)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ entity.getRelatesToEntities().clear();
+ }
+ }
+
+ // fetch info
+ boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
+ if (checkInfo &&
+ !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.INFO)) {
+ entity.getInfo().clear();
+ }
+ }
+
+ // fetch configs
+ boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
+ readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
+ if (checkConfigs && !TimelineStorageUtils.matchFilters(
+ entity.getConfigs(), configFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.CONFIGS)) {
+ entity.getConfigs().clear();
+ }
+ }
+
+ // fetch events
+ boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
+ readEvents(entity, result, false);
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(
+ entity.getEvents(), eventFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.EVENTS)) {
+ entity.getEvents().clear();
+ }
+ }
+
+ // fetch metrics
+ boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ if (fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ readMetrics(entity, result, EntityColumnPrefix.METRIC);
+ if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
+ entity.getMetrics(), metricFilters)) {
+ return null;
+ }
+ if (!fieldsToRetrieve.contains(Field.ALL) &&
+ !fieldsToRetrieve.contains(Field.METRICS)) {
+ entity.getMetrics().clear();
+ }
+ }
+ return entity;
+ }
+
+ /**
+ * Helper method for reading relationship.
+ */
+ protected <T> void readRelationship(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isRelatedTo) throws IOException {
+ // isRelatedTo and relatesTo are of type Map<String, Set<String>>
+ Map<String, Object> columns = prefix.readResults(result);
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ for (String id : Separator.VALUES.splitEncoded(
+ column.getValue().toString())) {
+ if (isRelatedTo) {
+ entity.addIsRelatedToEntity(column.getKey(), id);
+ } else {
+ entity.addRelatesToEntity(column.getKey(), id);
+ }
+ }
+ }
+ }
+
+ /**
+ * Helper method for reading key-value pairs for either info or config.
+ */
+ protected <T> void readKeyValuePairs(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isConfig) throws IOException {
+ // info and configuration are of type Map<String, Object or String>
+ Map<String, Object> columns = prefix.readResults(result);
+ if (isConfig) {
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ entity.addConfig(column.getKey(), column.getValue().toString());
+ }
+ } else {
+ entity.addInfo(columns);
+ }
+ }
+
+ /**
+ * Read events from the entity table or the application table. The column name
+ * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
+ * if there is no info associated with the event.
+ *
+ * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
+ * schema description.
+ */
+ protected void readEvents(TimelineEntity entity, Result result,
+ boolean isApplication) throws IOException {
+ Map<String, TimelineEvent> eventsMap = new HashMap<>();
+ Map<?, Object> eventsResult = isApplication ?
+ ApplicationColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result) :
+ EntityColumnPrefix.EVENT.
+ readResultsHavingCompoundColumnQualifiers(result);
+ for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
+ byte[][] karr = (byte[][])eventResult.getKey();
+ // the column name is of the form "eventId=timestamp=infoKey"
+ if (karr.length == 3) {
+ String id = Bytes.toString(karr[0]);
+ long ts = TimelineStorageUtils.invertLong(Bytes.toLong(karr[1]));
+ String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ // handle empty info
+ String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
+ if (infoKey != null) {
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ } else {
+ LOG.warn("incorrectly formatted column name: it will be discarded");
+ continue;
+ }
+ }
+ Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
+ entity.addEvents(eventsSet);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
new file mode 100644
index 0000000..e801466
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.NavigableSet;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.filter.FilterList;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
+
+/**
+ * The base class for reading and deserializing timeline entities from the
+ * HBase storage. Different types can be defined for different types of the
+ * entities that are being requested.
+ */
+public abstract class TimelineEntityReader {
+ private static final Log LOG = LogFactory.getLog(TimelineEntityReader.class);
+ protected static final long DEFAULT_BEGIN_TIME = 0L;
+ protected static final long DEFAULT_END_TIME = Long.MAX_VALUE;
+
+ protected final boolean singleEntityRead;
+
+ protected String userId;
+ protected String clusterId;
+ protected String flowName;
+ protected Long flowRunId;
+ protected String appId;
+ protected String entityType;
+ protected EnumSet<Field> fieldsToRetrieve;
+ // used only for a single entity read mode
+ protected String entityId;
+ // used only for multiple entity read mode
+ protected Long limit;
+ protected Long createdTimeBegin;
+ protected Long createdTimeEnd;
+ protected Long modifiedTimeBegin;
+ protected Long modifiedTimeEnd;
+ protected Map<String, Set<String>> relatesTo;
+ protected Map<String, Set<String>> isRelatedTo;
+ protected Map<String, Object> infoFilters;
+ protected Map<String, String> configFilters;
+ protected Set<String> metricFilters;
+ protected Set<String> eventFilters;
+ protected TimelineFilterList confsToRetrieve;
+ protected TimelineFilterList metricsToRetrieve;
+
+ /**
+ * Main table the entity reader uses.
+ */
+ protected BaseTable<?> table;
+
+ /**
+ * Specifies whether keys for this table are sorted in a manner where entities
+ * can be retrieved by created time. If true, it will be sufficient to collect
+ * the first results as specified by the limit. Otherwise all matched entities
+ * will be fetched and then limit applied.
+ */
+ private boolean sortedKeys = false;
+
+ /**
+ * Instantiates a reader for multiple-entity reads.
+ */
+ protected TimelineEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
+ EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
+ this.singleEntityRead = false;
+ this.sortedKeys = sortedKeys;
+ this.userId = userId;
+ this.clusterId = clusterId;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ this.entityType = entityType;
+ this.fieldsToRetrieve = fieldsToRetrieve;
+ this.limit = limit;
+ this.createdTimeBegin = createdTimeBegin;
+ this.createdTimeEnd = createdTimeEnd;
+ this.modifiedTimeBegin = modifiedTimeBegin;
+ this.modifiedTimeEnd = modifiedTimeEnd;
+ this.relatesTo = relatesTo;
+ this.isRelatedTo = isRelatedTo;
+ this.infoFilters = infoFilters;
+ this.configFilters = configFilters;
+ this.metricFilters = metricFilters;
+ this.eventFilters = eventFilters;
+ this.confsToRetrieve = confsToRetrieve;
+ this.metricsToRetrieve = metricsToRetrieve;
+
+ this.table = getTable();
+ }
+
+ /**
+ * Instantiates a reader for single-entity reads.
+ */
+ protected TimelineEntityReader(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId, String entityType,
+ String entityId, TimelineFilterList confsToRetrieve,
+ TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
+ this.singleEntityRead = true;
+ this.userId = userId;
+ this.clusterId = clusterId;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ this.entityType = entityType;
+ this.fieldsToRetrieve = fieldsToRetrieve;
+ this.entityId = entityId;
+ this.confsToRetrieve = confsToRetrieve;
+ this.metricsToRetrieve = metricsToRetrieve;
+
+ this.table = getTable();
+ }
+
+ /**
+ * Creates a {@link FilterList} based on fields, confs and metrics to
+ * retrieve. This filter list will be set in Scan/Get objects to trim down
+ * results fetched from HBase back-end storage.
+ * @return a {@link FilterList} object.
+ */
+ protected abstract FilterList constructFilterListBasedOnFields();
+
+ /**
+ * Reads and deserializes a single timeline entity from the HBase storage.
+ */
+ public TimelineEntity readEntity(Configuration hbaseConf, Connection conn)
+ throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ FilterList filterList = constructFilterListBasedOnFields();
+ Result result = getResult(hbaseConf, conn, filterList);
+ if (result == null || result.isEmpty()) {
+ // Could not find a matching row.
+ LOG.info("Cannot find matching entity of type " + entityType);
+ return null;
+ }
+ return parseEntity(result);
+ }
+
+ /**
+ * Reads and deserializes a set of timeline entities from the HBase storage.
+ * It goes through all the results available, and returns the number of
+ * entries as specified in the limit in the entity's natural sort order.
+ */
+ public Set<TimelineEntity> readEntities(Configuration hbaseConf,
+ Connection conn) throws IOException {
+ validateParams();
+ augmentParams(hbaseConf, conn);
+
+ NavigableSet<TimelineEntity> entities = new TreeSet<>();
+ FilterList filterList = constructFilterListBasedOnFields();
+ ResultScanner results = getResults(hbaseConf, conn, filterList);
+ try {
+ for (Result result : results) {
+ TimelineEntity entity = parseEntity(result);
+ if (entity == null) {
+ continue;
+ }
+ entities.add(entity);
+ if (!sortedKeys) {
+ if (entities.size() > limit) {
+ entities.pollLast();
+ }
+ } else {
+ if (entities.size() == limit) {
+ break;
+ }
+ }
+ }
+ return entities;
+ } finally {
+ results.close();
+ }
+ }
+
+ /**
+ * Returns the main table to be used by the entity reader.
+ */
+ protected abstract BaseTable<?> getTable();
+
+ /**
+ * Validates the required parameters to read the entities.
+ */
+ protected abstract void validateParams();
+
+ /**
+ * Sets certain parameters to defaults if the values are not provided.
+ */
+ protected abstract void augmentParams(Configuration hbaseConf,
+ Connection conn) throws IOException;
+
+ /**
+ * Fetches a {@link Result} instance for a single-entity read.
+ *
+ * @return the {@link Result} instance or null if no such record is found.
+ */
+ protected abstract Result getResult(Configuration hbaseConf, Connection conn,
+ FilterList filterList) throws IOException;
+
+ /**
+ * Fetches a {@link ResultScanner} for a multi-entity read.
+ */
+ protected abstract ResultScanner getResults(Configuration hbaseConf,
+ Connection conn, FilterList filterList) throws IOException;
+
+ /**
+ * Given a {@link Result} instance, deserializes and creates a
+ * {@link TimelineEntity}.
+ *
+ * @return the {@link TimelineEntity} instance, or null if the {@link Result}
+ * is null or empty.
+ */
+ protected abstract TimelineEntity parseEntity(Result result)
+ throws IOException;
+
+ /**
+ * Helper method for reading and deserializing {@link TimelineMetric} objects
+ * using the specified column prefix. The timeline metrics then are added to
+ * the given timeline entity.
+ */
+ protected void readMetrics(TimelineEntity entity, Result result,
+ ColumnPrefix<?> columnPrefix) throws IOException {
+ NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
+ columnPrefix.readResultsWithTimestamps(result);
+ for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
+ metricsResult.entrySet()) {
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(metricResult.getKey());
+ // Simply assume that if the value set contains more than 1 elements, the
+ // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
+ metric.setType(metricResult.getValue().size() > 1 ?
+ TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE);
+ metric.addValues(metricResult.getValue());
+ entity.addMetric(metric);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
new file mode 100644
index 0000000..c77897a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import java.util.EnumSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+
+/**
+ * Factory methods for instantiating a timeline entity reader.
+ */
+public class TimelineEntityReaderFactory {
+ /**
+ * Creates a timeline entity reader instance for reading a single entity with
+ * the specified input.
+ */
+ public static TimelineEntityReader createSingleEntityReader(String userId,
+ String clusterId, String flowName, Long flowRunId, String appId,
+ String entityType, String entityId, TimelineFilterList confs,
+ TimelineFilterList metrics, EnumSet<Field> fieldsToRetrieve) {
+ // currently the types that are handled separate from the generic entity
+ // table are application, flow run, and flow activity entities
+ if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
+ return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
+ return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
+ return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, entityId, fieldsToRetrieve);
+ } else {
+ // assume we're dealing with a generic entity read
+ return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
+ }
+ }
+
+ /**
+ * Creates a timeline entity reader instance for reading set of entities with
+ * the specified input and predicates.
+ */
+ public static TimelineEntityReader createMultipleEntitiesReader(String userId,
+ String clusterId, String flowName, Long flowRunId, String appId,
+ String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
+ Long modifiedTimeBegin, Long modifiedTimeEnd,
+ Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
+ Map<String, Object> infoFilters, Map<String, String> configFilters,
+ Set<String> metricFilters, Set<String> eventFilters,
+ TimelineFilterList confs, TimelineFilterList metrics,
+ EnumSet<Field> fieldsToRetrieve) {
+ // currently the types that are handled separate from the generic entity
+ // table are application, flow run, and flow activity entities
+ if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
+ return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters, confs,
+ metrics, fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
+ return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters,
+ fieldsToRetrieve);
+ } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
+ return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters, confs,
+ metrics, fieldsToRetrieve);
+ } else {
+ // assume we're dealing with a generic entity read
+ return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
+ appId, entityType, limit, createdTimeBegin, createdTimeEnd,
+ modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
+ infoFilters, configFilters, metricFilters, eventFilters, confs,
+ metrics, fieldsToRetrieve, false);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
new file mode 100644
index 0000000..0b3fa38
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-4446. Refactor reader API for
better extensibility (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index 0eeb195..ccb33b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -20,17 +20,14 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
/** ATSv2 reader interface. */
@Private
@@ -38,11 +35,6 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefi
public interface TimelineReader extends Service {
/**
- * Default limit for {@link #getEntities}.
- */
- long DEFAULT_LIMIT = 100;
-
- /**
* Possible fields to retrieve for {@link #getEntities} and
* {@link #getEntity}.
*/
@@ -57,55 +49,61 @@ public interface TimelineReader extends Service {
}
/**
- * <p>The API to fetch the single entity given the entity identifier in the
- * scope of the given context.</p>
- *
- * @param userId
- * Context user Id(optional).
- * @param clusterId
- * Context cluster Id(mandatory).
- * @param flowName
- * Context flow Id (optional).
- * @param flowRunId
- * Context flow run Id (optional).
- * @param appId
- * Context app Id (mandatory)
- * @param entityType
- * Entity type (mandatory)
- * @param entityId
- * Entity Id (mandatory)
- * @param confsToRetrieve
- * Used for deciding which configs to return in response. This is
- * represented as a {@link TimelineFilterList} object containing
- * {@link TimelinePrefixFilter} objects. These can either be exact config
- * keys' or prefixes which are then compared against config keys' to decide
- * configs to return in response.
- * @param metricsToRetrieve
- * Used for deciding which metrics to return in response. This is
- * represented as a {@link TimelineFilterList} object containing
- * {@link TimelinePrefixFilter} objects. These can either be exact metric
- * ids' or prefixes which are then compared against metric ids' to decide
- * metrics to return in response.
- * @param fieldsToRetrieve
- * Specifies which fields of the entity object to retrieve(optional), see
- * {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type and entity created time. All fields will be returned if
- * {@link Field#ALL} is specified.
- * @return a {@link TimelineEntity} instance or null. The entity will
- * contain the metadata plus the given fields to retrieve.
+ * <p>The API to fetch the single entity given the identifier(depending on
+ * the entity type) in the scope of the given context.</p>
+ * @param context Context which defines the scope in which query has to be
+ * made. Use getters of {@link TimelineReaderContext} to fetch context
+ * fields. Context contains the following :<br>
+ * <ul>
+ * <li><b>entityType</b> - Entity type(mandatory).</li>
+ * <li><b>clusterId</b> - Identifies the cluster(mandatory).</li>
+ * <li><b>userId</b> - Identifies the user.</li>
+ * <li><b>flowName</b> - Context flow name.</li>
+ * <li><b>flowRunId</b> - Context flow run id.</li>
+ * <li><b>appId</b> - Context app id.</li>
+ * <li><b>entityId</b> - Entity id.</li>
+ * </ul>
+ * Fields in context which are mandatory depends on entity type. Entity
+ * type is always mandatory. In addition to entity type, below is the list
+ * of context fields which are mandatory, based on entity type.<br>
+ * <ul>
+ * <li>If entity type is YARN_FLOW_RUN (i.e. query to fetch a specific flow
+ * run), clusterId, userId, flowName and flowRunId are mandatory.</li>
+ * <li>If entity type is YARN_APPLICATION (i.e. query to fetch a specific
+ * app), query is within the scope of clusterId, userId, flowName,
+ * flowRunId and appId. But out of this, only clusterId and appId are
+ * mandatory. If only clusterId and appId are supplied, backend storage
+ * must fetch the flow context information i.e. userId, flowName and
+ * flowRunId first and based on that, fetch the app. If flow context
+ * information is also given, app can be directly fetched.
+ * </li>
+ * <li>For other entity types (i.e. query to fetch generic entity), query
+ * is within the scope of clusterId, userId, flowName, flowRunId, appId,
+ * entityType and entityId. But out of this, only clusterId, appId,
+ * entityType and entityId are mandatory. If flow context information is
+ * not supplied, backend storage must fetch the flow context information
+ * i.e. userId, flowName and flowRunId first and based on that, fetch the
+ * entity. If flow context information is also given, entity can be
+ * directly queried.
+ * </li>
+ * </ul>
+ * @param dataToRetrieve Specifies which data to retrieve for the entity. Use
+ * getters of TimelineDataToRetrieve class to fetch dataToRetrieve
+ * fields. All the dataToRetrieve fields are optional. Refer to
+ * {@link TimelineDataToRetrieve} for details.
+ * @return A <cite>TimelineEntity</cite> instance or null. The entity will
+ * contain the metadata plus the given fields to retrieve.<br>
* If entityType is YARN_FLOW_RUN, entity returned is of type
- * {@link FlowRunEntity}.
+ * <cite>FlowRunEntity</cite>.<br>
* For all other entity types, entity returned is of type
- * {@link TimelineEntity}.
+ * <cite>TimelineEntity</cite>.
* @throws IOException
*/
- TimelineEntity getEntity(String userId, String clusterId, String flowName,
- Long flowRunId, String appId, String entityType, String entityId,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) throws IOException;
+ TimelineEntity getEntity(TimelineReaderContext context,
+ TimelineDataToRetrieve dataToRetrieve) throws IOException;
/**
- * <p>The API to search for a set of entities of the given the entity type in
+ * <p>The API to search for a set of entities of the given entity type in
* the scope of the given context which matches the given predicates. The
* predicates include the created time window, limit to number of entities to
* be returned, and the entities can be filtered by checking whether they
@@ -115,84 +113,66 @@ public interface TimelineReader extends Service {
* related to other entities. For those parameters which have multiple
* entries, the qualified entity needs to meet all or them.</p>
*
- * @param userId
- * Context user Id(optional).
- * @param clusterId
- * Context cluster Id(mandatory).
- * @param flowName
- * Context flow Id (optional).
- * @param flowRunId
- * Context flow run Id (optional).
- * @param appId
- * Context app Id (mandatory)
- * @param entityType
- * Entity type (mandatory)
- * @param limit
- * A limit on the number of entities to return (optional). If null or <=0,
- * defaults to {@link #DEFAULT_LIMIT}.
- * @param createdTimeBegin
- * Matched entities should not be created before this timestamp (optional).
- * If null or <=0, defaults to 0.
- * @param createdTimeEnd
- * Matched entities should not be created after this timestamp (optional).
- * If null or <=0, defaults to {@link Long#MAX_VALUE}.
- * @param relatesTo
- * Matched entities should relate to given entities (optional).
- * @param isRelatedTo
- * Matched entities should be related to given entities (optional).
- * @param infoFilters
- * Matched entities should have exact matches to the given info represented
- * as key-value pairs (optional). If null or empty, the filter is not
- * applied.
- * @param configFilters
- * Matched entities should have exact matches to the given configs
- * represented as key-value pairs (optional). If null or empty, the filter
- * is not applied.
- * @param metricFilters
- * Matched entities should contain the given metrics (optional). If null
- * or empty, the filter is not applied.
- * @param eventFilters
- * Matched entities should contain the given events (optional). If null
- * or empty, the filter is not applied.
- * @param confsToRetrieve
- * Used for deciding which configs to return in response. This is
- * represented as a {@link TimelineFilterList} object containing
- * {@link TimelinePrefixFilter} objects. These can either be exact config
- * keys' or prefixes which are then compared against config keys' to decide
- * configs(inside entities) to return in response. This should not be
- * confused with configFilters which is used to decide which entities to
- * return instead.
- * @param metricsToRetrieve
- * Used for deciding which metrics to return in response. This is
- * represented as a {@link TimelineFilterList} object containing
- * {@link TimelinePrefixFilter} objects. These can either be exact metric
- * ids' or prefixes which are then compared against metric ids' to decide
- * metrics(inside entities) to return in response. This should not be
- * confused with metricFilters which is used to decide which entities to
- * return instead.
- * @param fieldsToRetrieve
- * Specifies which fields of the entity object to retrieve(optional), see
- * {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type and entity created time. All fields will be returned if
- * {@link Field#ALL} is specified.
- * @return A set of {@link TimelineEntity} instances of the given entity type
- * in the given context scope which matches the given predicates
+ * @param context Context which defines the scope in which query has to be
+ * made. Use getters of {@link TimelineReaderContext} to fetch context
+ * fields. Context contains the following :<br>
+ * <ul>
+ * <li><b>entityType</b> - Entity type(mandatory).</li>
+ * <li><b>clusterId</b> - Identifies the cluster(mandatory).</li>
+ * <li><b>userId</b> - Identifies the user.</li>
+ * <li><b>flowName</b> - Context flow name.</li>
+ * <li><b>flowRunId</b> - Context flow run id.</li>
+ * <li><b>appId</b> - Context app id.</li>
+ * </ul>
+ * Although entityId is also part of context, it has no meaning for
+ * getEntities.<br>
+ * Fields in context which are mandatory depends on entity type. Entity
+ * type is always mandatory. In addition to entity type, below is the list
+ * of context fields which are mandatory, based on entity type.<br>
+ * <ul>
+ * <li>If entity type is YARN_FLOW_ACTIVITY (i.e. query to fetch flows),
+ * only clusterId is mandatory.
+ * </li>
+ * <li>If entity type is YARN_FLOW_RUN (i.e. query to fetch flow runs),
+ * clusterId, userId and flowName are mandatory.</li>
+ * <li>If entity type is YARN_APPLICATION (i.e. query to fetch apps), we
+ * can either get all apps within the context of flow name or within the
+ * context of flow run. If apps are queried within the scope of flow name,
+ * clusterId, userId and flowName are supplied. If they are queried within
+ * the scope of flow run, clusterId, userId, flowName and flowRunId are
+ * supplied.</li>
+ * <li>For other entity types (i.e. query to fetch generic entities), query
+ * is within the scope of clusterId, userId, flowName, flowRunId, appId and
+ * entityType. But out of this, only clusterId, appId and entityType are
+ * mandatory. If flow context information is not supplied, backend storage
+ * must fetch the flow context information i.e. userId, flowName and
+ * flowRunId first and based on that, fetch the entities. If flow context
+ * information is also given, entities can be directly queried.
+ * </li>
+ * </ul>
+ * @param filters Specifies filters which restrict the number of entities
+ * to return. Use getters of TimelineEntityFilters class to fetch
+ * various filters. All the filters are optional. Refer to
+ * {@link TimelineEntityFilters} for details.
+ * @param dataToRetrieve Specifies which data to retrieve for each entity. Use
+ * getters of TimelineDataToRetrieve class to fetch dataToRetrieve
+ * fields. All the dataToRetrieve fields are optional. Refer to
+ * {@link TimelineDataToRetrieve} for details.
+ * @return A set of <cite>TimelineEntity</cite> instances of the given entity
+ * type in the given context scope which matches the given predicates
* ordered by created time, descending. Each entity will only contain the
* metadata(id, type and created time) plus the given fields to retrieve.
+ * <br>
* If entityType is YARN_FLOW_ACTIVITY, entities returned are of type
- * {@link FlowActivityEntity}.
+ * <cite>FlowActivityEntity</cite>.<br>
* If entityType is YARN_FLOW_RUN, entities returned are of type
- * {@link FlowRunEntity}.
+ * <cite>FlowRunEntity</cite>.<br>
* For all other entity types, entities returned are of type
- * {@link TimelineEntity}.
+ * <cite>TimelineEntity</cite>.
* @throws IOException
*/
- Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) throws IOException;
+ Set<TimelineEntity> getEntities(
+ TimelineReaderContext context,
+ TimelineEntityFilters filters,
+ TimelineDataToRetrieve dataToRetrieve) throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index 4d61076..387f7d7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
@@ -38,9 +36,10 @@ import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnFamily;
@@ -60,26 +59,14 @@ class ApplicationEntityReader extends GenericEntityReader {
private static final ApplicationTable APPLICATION_TABLE =
new ApplicationTable();
- public ApplicationEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
- configFilters, metricFilters, eventFilters, confsToRetrieve,
- metricsToRetrieve, fieldsToRetrieve, true);
+ public ApplicationEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve, true);
}
- public ApplicationEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
+ public ApplicationEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
}
/**
@@ -92,12 +79,13 @@ class ApplicationEntityReader extends GenericEntityReader {
@Override
protected FilterList constructFilterListBasedOnFields() {
FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
// Fetch all the columns.
- if (fieldsToRetrieve.contains(Field.ALL) &&
- (confsToRetrieve == null ||
- confsToRetrieve.getFilterList().isEmpty()) &&
- (metricsToRetrieve == null ||
- metricsToRetrieve.getFilterList().isEmpty())) {
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (dataToRetrieve.getConfsToRetrieve() == null ||
+ dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) &&
+ (dataToRetrieve.getMetricsToRetrieve() == null ||
+ dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
return list;
}
FilterList infoColFamilyList = new FilterList();
@@ -107,61 +95,70 @@ class ApplicationEntityReader extends GenericEntityReader {
new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
infoColFamilyList.addFilter(infoColumnFamily);
// Events not required.
- if (!fieldsToRetrieve.contains(Field.EVENTS) &&
- !fieldsToRetrieve.contains(Field.ALL) && eventFilters == null) {
+ TimelineEntityFilters filters = getFilters();
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.EVENTS) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getEventFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
ApplicationColumnPrefix.EVENT.getColumnPrefixBytes(""))));
}
// info not required.
- if (!fieldsToRetrieve.contains(Field.INFO) &&
- !fieldsToRetrieve.contains(Field.ALL) && infoFilters == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.INFO) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getInfoFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
ApplicationColumnPrefix.INFO.getColumnPrefixBytes(""))));
}
// is releated to not required.
- if (!fieldsToRetrieve.contains(Field.IS_RELATED_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && isRelatedTo == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.IS_RELATED_TO) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getIsRelatedTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
ApplicationColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
}
// relates to not required.
- if (!fieldsToRetrieve.contains(Field.RELATES_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && relatesTo == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.RELATES_TO) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getRelatesTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
ApplicationColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
}
list.addFilter(infoColFamilyList);
- if ((fieldsToRetrieve.contains(Field.CONFIGS) || configFilters != null) ||
- (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty())) {
+ if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS) ||
+ (!singleEntityRead && filters.getConfigFilters() != null)) ||
+ (dataToRetrieve.getConfsToRetrieve() != null &&
+ !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty())) {
FilterList filterCfg =
new FilterList(new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(ApplicationColumnFamily.CONFIGS.getBytes())));
- if (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty()) {
+ if (dataToRetrieve.getConfsToRetrieve() != null &&
+ !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) {
filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.CONFIG, confsToRetrieve));
+ ApplicationColumnPrefix.CONFIG,
+ dataToRetrieve.getConfsToRetrieve()));
}
list.addFilter(filterCfg);
}
- if ((fieldsToRetrieve.contains(Field.METRICS) || metricFilters != null) ||
- (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty())) {
+ if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) ||
+ (!singleEntityRead && filters.getMetricFilters() != null)) ||
+ (dataToRetrieve.getMetricsToRetrieve() != null &&
+ !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
FilterList filterMetrics =
new FilterList(new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(ApplicationColumnFamily.METRICS.getBytes())));
- if (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
+ if (dataToRetrieve.getMetricsToRetrieve() != null &&
+ !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty()) {
filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.METRIC, metricsToRetrieve));
+ ApplicationColumnPrefix.METRIC,
+ dataToRetrieve.getMetricsToRetrieve()));
}
list.addFilter(filterMetrics);
}
@@ -171,9 +168,10 @@ class ApplicationEntityReader extends GenericEntityReader {
@Override
protected Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
+ TimelineReaderContext context = getContext();
byte[] rowKey =
- ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
- appId);
+ ApplicationRowKey.getRowKey(context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId(), context.getAppId());
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
if (filterList != null && !filterList.getFilters().isEmpty()) {
@@ -184,66 +182,54 @@ class ApplicationEntityReader extends GenericEntityReader {
@Override
protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getEntityType(),
+ "entityType shouldn't be null");
if (singleEntityRead) {
- Preconditions.checkNotNull(appId, "appId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getAppId(),
+ "appId shouldn't be null");
} else {
- Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
+ Preconditions.checkNotNull(getContext().getUserId(),
+ "userId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getFlowName(),
+ "flowName shouldn't be null");
}
}
@Override
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
+ TimelineReaderContext context = getContext();
if (singleEntityRead) {
- if (flowName == null || flowRunId == null || userId == null) {
- FlowContext context =
- lookupFlowContext(clusterId, appId, hbaseConf, conn);
- flowName = context.flowName;
- flowRunId = context.flowRunId;
- userId = context.userId;
- }
- }
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
- if (!fieldsToRetrieve.contains(Field.CONFIGS) &&
- confsToRetrieve != null && !confsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.CONFIGS);
- }
- if (!fieldsToRetrieve.contains(Field.METRICS) &&
- metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.METRICS);
- }
- if (!singleEntityRead) {
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
+ if (context.getFlowName() == null || context.getFlowRunId() == null ||
+ context.getUserId() == null) {
+ FlowContext flowContext = lookupFlowContext(
+ context.getClusterId(), context.getAppId(), hbaseConf, conn);
+ context.setFlowName(flowContext.flowName);
+ context.setFlowRunId(flowContext.flowRunId);
+ context.setUserId(flowContext.userId);
}
}
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
}
@Override
protected ResultScanner getResults(Configuration hbaseConf,
Connection conn, FilterList filterList) throws IOException {
Scan scan = new Scan();
- if (flowRunId != null) {
+ TimelineReaderContext context = getContext();
+ if (context.getFlowRunId() != null) {
scan.setRowPrefixFilter(ApplicationRowKey.
- getRowKeyPrefix(clusterId, userId, flowName, flowRunId));
+ getRowKeyPrefix(context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId()));
} else {
scan.setRowPrefixFilter(ApplicationRowKey.
- getRowKeyPrefix(clusterId, userId, flowName));
+ getRowKeyPrefix(context.getClusterId(), context.getUserId(),
+ context.getFlowName()));
}
FilterList newList = new FilterList();
- newList.addFilter(new PageFilter(limit));
+ newList.addFilter(new PageFilter(getFilters().getLimit()));
if (filterList != null && !filterList.getFilters().isEmpty()) {
newList.addFilter(filterList);
}
@@ -261,23 +247,27 @@ class ApplicationEntityReader extends GenericEntityReader {
String entityId = ApplicationColumn.ID.readResult(result).toString();
entity.setId(entityId);
+ TimelineEntityFilters filters = getFilters();
// fetch created time
Number createdTime =
(Number)ApplicationColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue());
- if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
- entity.getCreatedTime() > createdTimeEnd)) {
+ if (!singleEntityRead &&
+ (entity.getCreatedTime() < filters.getCreatedTimeBegin() ||
+ entity.getCreatedTime() > filters.getCreatedTimeEnd())) {
return null;
}
-
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
// fetch is related to entities
- boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ boolean checkIsRelatedTo =
+ filters != null && filters.getIsRelatedTo() != null &&
+ filters.getIsRelatedTo().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
true);
if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), isRelatedTo)) {
+ entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -287,13 +277,15 @@ class ApplicationEntityReader extends GenericEntityReader {
}
// fetch relates to entities
- boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ boolean checkRelatesTo =
+ filters != null && filters.getRelatesTo() != null &&
+ filters.getRelatesTo().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
false);
if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), relatesTo)) {
+ entity.getRelatesToEntities(), filters.getRelatesTo())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -303,12 +295,14 @@ class ApplicationEntityReader extends GenericEntityReader {
}
// fetch info
- boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ boolean checkInfo = filters != null && filters.getInfoFilters() != null &&
+ filters.getInfoFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
if (checkInfo &&
- !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineStorageUtils.matchFilters(
+ entity.getInfo(), filters.getInfoFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -318,12 +312,14 @@ class ApplicationEntityReader extends GenericEntityReader {
}
// fetch configs
- boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ boolean checkConfigs =
+ filters != null && filters.getConfigFilters() != null &&
+ filters.getConfigFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
if (checkConfigs && !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), configFilters)) {
+ entity.getConfigs(), filters.getConfigFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -333,12 +329,14 @@ class ApplicationEntityReader extends GenericEntityReader {
}
// fetch events
- boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ boolean checkEvents =
+ filters != null && filters.getEventFilters() != null &&
+ filters.getEventFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
readEvents(entity, result, true);
if (checkEvents && !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), eventFilters)) {
+ entity.getEvents(), filters.getEventFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -348,12 +346,14 @@ class ApplicationEntityReader extends GenericEntityReader {
}
// fetch metrics
- boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ boolean checkMetrics =
+ filters != null && filters.getMetricFilters() != null &&
+ filters.getMetricFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), metricFilters)) {
+ entity.getMetrics(), filters.getMetricFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
index 048f608..96350da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -18,9 +18,7 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
-import java.util.EnumSet;
import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
@@ -32,8 +30,9 @@ import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
@@ -49,24 +48,14 @@ class FlowActivityEntityReader extends TimelineEntityReader {
private static final FlowActivityTable FLOW_ACTIVITY_TABLE =
new FlowActivityTable();
- public FlowActivityEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
- configFilters, metricFilters, eventFilters, null, null,
- fieldsToRetrieve, true);
+ public FlowActivityEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve, true);
}
- public FlowActivityEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- null, null, fieldsToRetrieve);
+ public FlowActivityEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
}
/**
@@ -79,21 +68,13 @@ class FlowActivityEntityReader extends TimelineEntityReader {
@Override
protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
}
@Override
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
}
@Override
@@ -112,20 +93,24 @@ class FlowActivityEntityReader extends TimelineEntityReader {
protected ResultScanner getResults(Configuration hbaseConf,
Connection conn, FilterList filterList) throws IOException {
Scan scan = new Scan();
- if (createdTimeBegin == DEFAULT_BEGIN_TIME &&
- createdTimeEnd == DEFAULT_END_TIME) {
+ String clusterId = getContext().getClusterId();
+ if (getFilters().getCreatedTimeBegin() == 0L &&
+ getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) {
+ // All records have to be chosen.
scan.setRowPrefixFilter(FlowActivityRowKey.getRowKeyPrefix(clusterId));
} else {
scan.setStartRow(
- FlowActivityRowKey.getRowKeyPrefix(clusterId, createdTimeEnd));
+ FlowActivityRowKey.getRowKeyPrefix(clusterId,
+ getFilters().getCreatedTimeEnd()));
scan.setStopRow(
FlowActivityRowKey.getRowKeyPrefix(clusterId,
- (createdTimeBegin <= 0 ? 0: (createdTimeBegin - 1))));
+ (getFilters().getCreatedTimeBegin() <= 0 ? 0 :
+ (getFilters().getCreatedTimeBegin() - 1))));
}
// use the page filter to limit the result to the page size
// the scanner may still return more than the limit; therefore we need to
// read the right number as we iterate
- scan.setFilter(new PageFilter(limit));
+ scan.setFilter(new PageFilter(getFilters().getLimit()));
return table.getResultScanner(hbaseConf, conn, scan);
}
@@ -137,8 +122,8 @@ class FlowActivityEntityReader extends TimelineEntityReader {
String user = rowKey.getUserId();
String flowName = rowKey.getFlowName();
- FlowActivityEntity flowActivity =
- new FlowActivityEntity(clusterId, time, user, flowName);
+ FlowActivityEntity flowActivity = new FlowActivityEntity(
+ getContext().getClusterId(), time, user, flowName);
// set the id
flowActivity.setId(flowActivity.getId());
// get the list of run ids along with the version that are associated with
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
index 4f50b02..2d1c41c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
@@ -38,9 +35,10 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
@@ -58,26 +56,14 @@ import com.google.common.base.Preconditions;
class FlowRunEntityReader extends TimelineEntityReader {
private static final FlowRunTable FLOW_RUN_TABLE = new FlowRunTable();
- public FlowRunEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
- configFilters, metricFilters, eventFilters, null, metricsToRetrieve,
- fieldsToRetrieve, true);
+ public FlowRunEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, entityFilters, toRetrieve, true);
}
- public FlowRunEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- null, metricsToRetrieve, fieldsToRetrieve);
+ public FlowRunEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
}
/**
@@ -90,35 +76,21 @@ class FlowRunEntityReader extends TimelineEntityReader {
@Override
protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getUserId(),
+ "userId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getFlowName(),
+ "flowName shouldn't be null");
if (singleEntityRead) {
- Preconditions.checkNotNull(flowRunId, "flowRunId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getFlowRunId(),
+ "flowRunId shouldn't be null");
}
}
@Override
protected void augmentParams(Configuration hbaseConf, Connection conn) {
- if (!singleEntityRead) {
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
- if (!fieldsToRetrieve.contains(Field.METRICS) &&
- metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.METRICS);
- }
- }
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
}
@Override
@@ -129,9 +101,11 @@ class FlowRunEntityReader extends TimelineEntityReader {
FamilyFilter infoColumnFamily =
new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(FlowRunColumnFamily.INFO.getBytes()));
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
// Metrics not required.
- if (!singleEntityRead && !fieldsToRetrieve.contains(Field.METRICS) &&
- !fieldsToRetrieve.contains(Field.ALL)) {
+ if (!singleEntityRead &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL)) {
FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
infoColFamilyList.addFilter(infoColumnFamily);
infoColFamilyList.addFilter(
@@ -140,12 +114,12 @@ class FlowRunEntityReader extends TimelineEntityReader {
FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(""))));
list.addFilter(infoColFamilyList);
}
- if (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
+ if (dataToRetrieve.getMetricsToRetrieve() != null &&
+ !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty()) {
FilterList infoColFamilyList = new FilterList();
infoColFamilyList.addFilter(infoColumnFamily);
infoColFamilyList.addFilter(TimelineFilterUtils.createHBaseFilterList(
- FlowRunColumnPrefix.METRIC, metricsToRetrieve));
+ FlowRunColumnPrefix.METRIC, dataToRetrieve.getMetricsToRetrieve()));
list.addFilter(infoColFamilyList);
}
return list;
@@ -154,8 +128,10 @@ class FlowRunEntityReader extends TimelineEntityReader {
@Override
protected Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
+ TimelineReaderContext context = getContext();
byte[] rowKey =
- FlowRunRowKey.getRowKey(clusterId, userId, flowName, flowRunId);
+ FlowRunRowKey.getRowKey(context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId());
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
if (filterList != null && !filterList.getFilters().isEmpty()) {
@@ -168,10 +144,12 @@ class FlowRunEntityReader extends TimelineEntityReader {
protected ResultScanner getResults(Configuration hbaseConf,
Connection conn, FilterList filterList) throws IOException {
Scan scan = new Scan();
+ TimelineReaderContext context = getContext();
scan.setRowPrefixFilter(
- FlowRunRowKey.getRowKeyPrefix(clusterId, userId, flowName));
+ FlowRunRowKey.getRowKeyPrefix(context.getClusterId(),
+ context.getUserId(), context.getFlowName()));
FilterList newList = new FilterList();
- newList.addFilter(new PageFilter(limit));
+ newList.addFilter(new PageFilter(getFilters().getLimit()));
if (filterList != null && !filterList.getFilters().isEmpty()) {
newList.addFilter(filterList);
}
@@ -181,11 +159,12 @@ class FlowRunEntityReader extends TimelineEntityReader {
@Override
protected TimelineEntity parseEntity(Result result) throws IOException {
+ TimelineReaderContext context = getContext();
FlowRunEntity flowRun = new FlowRunEntity();
- flowRun.setUser(userId);
- flowRun.setName(flowName);
+ flowRun.setUser(context.getUserId());
+ flowRun.setName(context.getFlowName());
if (singleEntityRead) {
- flowRun.setRunId(flowRunId);
+ flowRun.setRunId(context.getFlowRunId());
} else {
FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
flowRun.setRunId(rowKey.getFlowRunId());
@@ -196,8 +175,9 @@ class FlowRunEntityReader extends TimelineEntityReader {
if (startTime != null) {
flowRun.setStartTime(startTime.longValue());
}
- if (!singleEntityRead && (flowRun.getStartTime() < createdTimeBegin ||
- flowRun.getStartTime() > createdTimeEnd)) {
+ if (!singleEntityRead &&
+ (flowRun.getStartTime() < getFilters().getCreatedTimeBegin() ||
+ flowRun.getStartTime() > getFilters().getCreatedTimeEnd())) {
return null;
}
@@ -214,7 +194,8 @@ class FlowRunEntityReader extends TimelineEntityReader {
}
// read metrics
- if (singleEntityRead || fieldsToRetrieve.contains(Field.METRICS)) {
+ if (singleEntityRead ||
+ getDataToRetrieve().getFieldsToRetrieve().contains(Field.METRICS)) {
readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index 237b9ac..3bc2f3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -42,9 +42,10 @@ import org.apache.hadoop.hbase.filter.FilterList.Operator;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
@@ -77,26 +78,15 @@ class GenericEntityReader extends TimelineEntityReader {
*/
private final AppToFlowTable appToFlowTable = new AppToFlowTable();
- public GenericEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo, infoFilters,
- configFilters, metricFilters, eventFilters, confsToRetrieve,
- metricsToRetrieve, fieldsToRetrieve, sortedKeys);
+ public GenericEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve,
+ boolean sortedKeys) {
+ super(ctxt, entityFilters, toRetrieve, sortedKeys);
}
- public GenericEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
+ public GenericEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
+ super(ctxt, toRetrieve);
}
/**
@@ -109,12 +99,13 @@ class GenericEntityReader extends TimelineEntityReader {
@Override
protected FilterList constructFilterListBasedOnFields() {
FilterList list = new FilterList(Operator.MUST_PASS_ONE);
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
// Fetch all the columns.
- if (fieldsToRetrieve.contains(Field.ALL) &&
- (confsToRetrieve == null ||
- confsToRetrieve.getFilterList().isEmpty()) &&
- (metricsToRetrieve == null ||
- metricsToRetrieve.getFilterList().isEmpty())) {
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (dataToRetrieve.getConfsToRetrieve() == null ||
+ dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) &&
+ (dataToRetrieve.getMetricsToRetrieve() == null ||
+ dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
return list;
}
FilterList infoColFamilyList = new FilterList();
@@ -123,62 +114,69 @@ class GenericEntityReader extends TimelineEntityReader {
new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(EntityColumnFamily.INFO.getBytes()));
infoColFamilyList.addFilter(infoColumnFamily);
+ TimelineEntityFilters filters = getFilters();
// Events not required.
- if (!fieldsToRetrieve.contains(Field.EVENTS) &&
- !fieldsToRetrieve.contains(Field.ALL) && eventFilters == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.EVENTS) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getEventFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
EntityColumnPrefix.EVENT.getColumnPrefixBytes(""))));
}
// info not required.
- if (!fieldsToRetrieve.contains(Field.INFO) &&
- !fieldsToRetrieve.contains(Field.ALL) && infoFilters == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.INFO) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getInfoFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
EntityColumnPrefix.INFO.getColumnPrefixBytes(""))));
}
// is related to not required.
- if (!fieldsToRetrieve.contains(Field.IS_RELATED_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && isRelatedTo == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.IS_RELATED_TO) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getIsRelatedTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
EntityColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
}
// relates to not required.
- if (!fieldsToRetrieve.contains(Field.RELATES_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && relatesTo == null) {
+ if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.RELATES_TO) &&
+ !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
+ (singleEntityRead || filters.getRelatesTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
EntityColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
}
list.addFilter(infoColFamilyList);
- if ((fieldsToRetrieve.contains(Field.CONFIGS) || configFilters != null) ||
- (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty())) {
+ if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS) ||
+ (!singleEntityRead && filters.getConfigFilters() != null)) ||
+ (dataToRetrieve.getConfsToRetrieve() != null &&
+ !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty())) {
FilterList filterCfg =
new FilterList(new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(EntityColumnFamily.CONFIGS.getBytes())));
- if (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty()) {
+ if (dataToRetrieve.getConfsToRetrieve() != null &&
+ !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) {
filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.CONFIG, confsToRetrieve));
+ EntityColumnPrefix.CONFIG, dataToRetrieve.getConfsToRetrieve()));
}
list.addFilter(filterCfg);
}
- if ((fieldsToRetrieve.contains(Field.METRICS) || metricFilters != null) ||
- (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty())) {
+ if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) ||
+ (!singleEntityRead && filters.getMetricFilters() != null)) ||
+ (dataToRetrieve.getMetricsToRetrieve() != null &&
+ !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
FilterList filterMetrics =
new FilterList(new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(EntityColumnFamily.METRICS.getBytes())));
- if (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
+ if (dataToRetrieve.getMetricsToRetrieve() != null &&
+ !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty()) {
filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.METRIC, metricsToRetrieve));
+ EntityColumnPrefix.METRIC, dataToRetrieve.getMetricsToRetrieve()));
}
list.addFilter(filterMetrics);
}
@@ -215,56 +213,42 @@ class GenericEntityReader extends TimelineEntityReader {
@Override
protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(appId, "appId shouldn't be null");
- Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
+ Preconditions.checkNotNull(getContext().getClusterId(),
+ "clusterId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getAppId(),
+ "appId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getEntityType(),
+ "entityType shouldn't be null");
if (singleEntityRead) {
- Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
+ Preconditions.checkNotNull(getContext().getEntityId(),
+ "entityId shouldn't be null");
}
}
@Override
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
+ TimelineReaderContext context = getContext();
// In reality all three should be null or neither should be null
- if (flowName == null || flowRunId == null || userId == null) {
- FlowContext context =
- lookupFlowContext(clusterId, appId, hbaseConf, conn);
- flowName = context.flowName;
- flowRunId = context.flowRunId;
- userId = context.userId;
- }
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
- if (!fieldsToRetrieve.contains(Field.CONFIGS) &&
- confsToRetrieve != null && !confsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.CONFIGS);
- }
- if (!fieldsToRetrieve.contains(Field.METRICS) &&
- metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.METRICS);
- }
- if (!singleEntityRead) {
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
+ if (context.getFlowName() == null || context.getFlowRunId() == null ||
+ context.getUserId() == null) {
+ FlowContext flowContext = lookupFlowContext(
+ context.getClusterId(), context.getAppId(), hbaseConf, conn);
+ context.setFlowName(flowContext.flowName);
+ context.setFlowRunId(flowContext.flowRunId);
+ context.setUserId(flowContext.userId);
}
+ getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
}
@Override
protected Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException {
+ TimelineReaderContext context = getContext();
byte[] rowKey =
- EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
- entityType, entityId);
+ EntityRowKey.getRowKey(context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId(), context.getAppId(),
+ context.getEntityType(), context.getEntityId());
Get get = new Get(rowKey);
get.setMaxVersions(Integer.MAX_VALUE);
if (filterList != null && !filterList.getFilters().isEmpty()) {
@@ -279,8 +263,10 @@ class GenericEntityReader extends TimelineEntityReader {
// Scan through part of the table to find the entities belong to one app
// and one type
Scan scan = new Scan();
+ TimelineReaderContext context = getContext();
scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
- clusterId, userId, flowName, flowRunId, appId, entityType));
+ context.getClusterId(), context.getUserId(), context.getFlowName(),
+ context.getFlowRunId(), context.getAppId(), context.getEntityType()));
scan.setMaxVersions(Integer.MAX_VALUE);
if (filterList != null && !filterList.getFilters().isEmpty()) {
scan.setFilter(filterList);
@@ -299,21 +285,25 @@ class GenericEntityReader extends TimelineEntityReader {
String entityId = EntityColumn.ID.readResult(result).toString();
entity.setId(entityId);
+ TimelineEntityFilters filters = getFilters();
// fetch created time
Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue());
- if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
- entity.getCreatedTime() > createdTimeEnd)) {
+ if (!singleEntityRead &&
+ (entity.getCreatedTime() < filters.getCreatedTimeBegin() ||
+ entity.getCreatedTime() > filters.getCreatedTimeEnd())) {
return null;
}
-
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
// fetch is related to entities
- boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
+ boolean checkIsRelatedTo =
+ filters != null && filters.getIsRelatedTo() != null &&
+ filters.getIsRelatedTo().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), isRelatedTo)) {
+ entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -323,12 +313,14 @@ class GenericEntityReader extends TimelineEntityReader {
}
// fetch relates to entities
- boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
+ boolean checkRelatesTo =
+ filters != null && filters.getRelatesTo() != null &&
+ filters.getRelatesTo().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), relatesTo)) {
+ entity.getRelatesToEntities(), filters.getRelatesTo())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -338,12 +330,14 @@ class GenericEntityReader extends TimelineEntityReader {
}
// fetch info
- boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
+ boolean checkInfo = filters != null && filters.getInfoFilters() != null &&
+ filters.getInfoFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
if (checkInfo &&
- !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ !TimelineStorageUtils.matchFilters(
+ entity.getInfo(), filters.getInfoFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -353,12 +347,14 @@ class GenericEntityReader extends TimelineEntityReader {
}
// fetch configs
- boolean checkConfigs = configFilters != null && configFilters.size() > 0;
+ boolean checkConfigs =
+ filters != null && filters.getConfigFilters() != null &&
+ filters.getConfigFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
if (checkConfigs && !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), configFilters)) {
+ entity.getConfigs(), filters.getConfigFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -368,12 +364,14 @@ class GenericEntityReader extends TimelineEntityReader {
}
// fetch events
- boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
+ boolean checkEvents =
+ filters != null && filters.getEventFilters() != null &&
+ filters.getEventFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
readEvents(entity, result, false);
if (checkEvents && !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), eventFilters)) {
+ entity.getEvents(), filters.getEventFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
@@ -383,12 +381,14 @@ class GenericEntityReader extends TimelineEntityReader {
}
// fetch metrics
- boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
+ boolean checkMetrics =
+ filters != null && filters.getMetricFilters() != null &&
+ filters.getMetricFilters().size() > 0;
if (fieldsToRetrieve.contains(Field.ALL) ||
fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
readMetrics(entity, result, EntityColumnPrefix.METRIC);
if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), metricFilters)) {
+ entity.getMetrics(), filters.getMetricFilters())) {
return null;
}
if (!fieldsToRetrieve.contains(Field.ALL) &&
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
index bc86b6d..454c179 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
-import java.util.EnumSet;
import java.util.Map;
import java.util.NavigableMap;
import java.util.NavigableSet;
@@ -34,8 +33,9 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
@@ -46,32 +46,12 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
*/
public abstract class TimelineEntityReader {
private static final Log LOG = LogFactory.getLog(TimelineEntityReader.class);
- protected static final long DEFAULT_BEGIN_TIME = 0L;
- protected static final long DEFAULT_END_TIME = Long.MAX_VALUE;
protected final boolean singleEntityRead;
-
- protected String userId;
- protected String clusterId;
- protected String flowName;
- protected Long flowRunId;
- protected String appId;
- protected String entityType;
- protected EnumSet<Field> fieldsToRetrieve;
- // used only for a single entity read mode
- protected String entityId;
+ private TimelineReaderContext context;
+ private TimelineDataToRetrieve dataToRetrieve;
// used only for multiple entity read mode
- protected Long limit;
- protected Long createdTimeBegin;
- protected Long createdTimeEnd;
- protected Map<String, Set<String>> relatesTo;
- protected Map<String, Set<String>> isRelatedTo;
- protected Map<String, Object> infoFilters;
- protected Map<String, String> configFilters;
- protected Set<String> metricFilters;
- protected Set<String> eventFilters;
- protected TimelineFilterList confsToRetrieve;
- protected TimelineFilterList metricsToRetrieve;
+ private TimelineEntityFilters filters;
/**
* Main table the entity reader uses.
@@ -89,34 +69,14 @@ public abstract class TimelineEntityReader {
/**
* Instantiates a reader for multiple-entity reads.
*/
- protected TimelineEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
+ protected TimelineEntityReader(TimelineReaderContext ctxt,
+ TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve,
+ boolean sortedKeys) {
this.singleEntityRead = false;
this.sortedKeys = sortedKeys;
- this.userId = userId;
- this.clusterId = clusterId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- this.appId = appId;
- this.entityType = entityType;
- this.fieldsToRetrieve = fieldsToRetrieve;
- this.limit = limit;
- this.createdTimeBegin = createdTimeBegin;
- this.createdTimeEnd = createdTimeEnd;
- this.relatesTo = relatesTo;
- this.isRelatedTo = isRelatedTo;
- this.infoFilters = infoFilters;
- this.configFilters = configFilters;
- this.metricFilters = metricFilters;
- this.eventFilters = eventFilters;
- this.confsToRetrieve = confsToRetrieve;
- this.metricsToRetrieve = metricsToRetrieve;
+ this.context = ctxt;
+ this.dataToRetrieve = toRetrieve;
+ this.filters = entityFilters;
this.table = getTable();
}
@@ -124,21 +84,11 @@ public abstract class TimelineEntityReader {
/**
* Instantiates a reader for single-entity reads.
*/
- protected TimelineEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
+ protected TimelineEntityReader(TimelineReaderContext ctxt,
+ TimelineDataToRetrieve toRetrieve) {
this.singleEntityRead = true;
- this.userId = userId;
- this.clusterId = clusterId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- this.appId = appId;
- this.entityType = entityType;
- this.fieldsToRetrieve = fieldsToRetrieve;
- this.entityId = entityId;
- this.confsToRetrieve = confsToRetrieve;
- this.metricsToRetrieve = metricsToRetrieve;
+ this.context = ctxt;
+ this.dataToRetrieve = toRetrieve;
this.table = getTable();
}
@@ -151,6 +101,18 @@ public abstract class TimelineEntityReader {
*/
protected abstract FilterList constructFilterListBasedOnFields();
+ protected TimelineReaderContext getContext() {
+ return context;
+ }
+
+ protected TimelineDataToRetrieve getDataToRetrieve() {
+ return dataToRetrieve;
+ }
+
+ protected TimelineEntityFilters getFilters() {
+ return filters;
+ }
+
/**
* Reads and deserializes a single timeline entity from the HBase storage.
*/
@@ -163,7 +125,8 @@ public abstract class TimelineEntityReader {
Result result = getResult(hbaseConf, conn, filterList);
if (result == null || result.isEmpty()) {
// Could not find a matching row.
- LOG.info("Cannot find matching entity of type " + entityType);
+ LOG.info("Cannot find matching entity of type " +
+ context.getEntityType());
return null;
}
return parseEntity(result);
@@ -190,11 +153,11 @@ public abstract class TimelineEntityReader {
}
entities.add(entity);
if (!sortedKeys) {
- if (entities.size() > limit) {
+ if (entities.size() > filters.getLimit()) {
entities.pollLast();
}
} else {
- if (entities.size() == limit) {
+ if (entities.size() == filters.getLimit()) {
break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
index 2e2c652..f2bdacd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -17,13 +17,10 @@
*/
package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
/**
* Factory methods for instantiating a timeline entity reader.
@@ -33,25 +30,21 @@ public class TimelineEntityReaderFactory {
* Creates a timeline entity reader instance for reading a single entity with
* the specified input.
*/
- public static TimelineEntityReader createSingleEntityReader(String userId,
- String clusterId, String flowName, Long flowRunId, String appId,
- String entityType, String entityId, TimelineFilterList confs,
- TimelineFilterList metrics, EnumSet<Field> fieldsToRetrieve) {
+ public static TimelineEntityReader createSingleEntityReader(
+ TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) {
// currently the types that are handled separate from the generic entity
// table are application, flow run, and flow activity entities
- if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
- return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
- return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
- return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, fieldsToRetrieve);
+ if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
+ return new ApplicationEntityReader(context, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_RUN.matches(context.getEntityType())) {
+ return new FlowRunEntityReader(context, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
+ return new FlowActivityEntityReader(context, dataToRetrieve);
} else {
// assume we're dealing with a generic entity read
- return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
+ return new GenericEntityReader(context, dataToRetrieve);
}
}
@@ -59,37 +52,22 @@ public class TimelineEntityReaderFactory {
* Creates a timeline entity reader instance for reading set of entities with
* the specified input and predicates.
*/
- public static TimelineEntityReader createMultipleEntitiesReader(String userId,
- String clusterId, String flowName, Long flowRunId, String appId,
- String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confs, TimelineFilterList metrics,
- EnumSet<Field> fieldsToRetrieve) {
+ public static TimelineEntityReader createMultipleEntitiesReader(
+ TimelineReaderContext context, TimelineEntityFilters filters,
+ TimelineDataToRetrieve dataToRetrieve) {
// currently the types that are handled separate from the generic entity
// table are application, flow run, and flow activity entities
- if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
- return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
- isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
- confs, metrics, fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
- return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
- isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
- fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
- return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
- isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
- confs, metrics, fieldsToRetrieve);
+ if (TimelineEntityType.YARN_APPLICATION.matches(context.getEntityType())) {
+ return new ApplicationEntityReader(context, filters, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_ACTIVITY.matches(context.getEntityType())) {
+ return new FlowActivityEntityReader(context, filters, dataToRetrieve);
+ } else if (TimelineEntityType.
+ YARN_FLOW_RUN.matches(context.getEntityType())) {
+ return new FlowRunEntityReader(context, filters, dataToRetrieve);
} else {
// assume we're dealing with a generic entity read
- return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
- isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
- confs, metrics, fieldsToRetrieve, false);
+ return new GenericEntityReader(context, filters, dataToRetrieve, false);
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters
in TimelineReader (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index b5fc214..2d85bab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -17,21 +17,26 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
-import java.io.IOException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -39,6 +44,15 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter.TimelineFilterType;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
@@ -53,6 +67,8 @@ public final class TimelineStorageUtils {
private TimelineStorageUtils() {
}
+ private static final Log LOG = LogFactory.getLog(TimelineStorageUtils.class);
+
/** empty bytes. */
public static final byte[] EMPTY_BYTES = new byte[0];
@@ -312,6 +328,21 @@ public final class TimelineStorageUtils {
}
/**
+ * Check if we have a certain field amongst fields to retrieve. This method
+ * checks against {@link Field#ALL} as well because that would mean field
+ * passed needs to be matched.
+ *
+ * @param fieldsToRetrieve fields to be retrieved.
+ * @param requiredField fields to be checked in fieldsToRetrieve.
+ * @return true if has the required field, false otherwise.
+ */
+ public static boolean hasField(EnumSet<Field> fieldsToRetrieve,
+ Field requiredField) {
+ return fieldsToRetrieve.contains(Field.ALL) ||
+ fieldsToRetrieve.contains(requiredField);
+ }
+
+ /**
* Checks if the input TimelineEntity object is an ApplicationEntity.
*
* @param te TimelineEntity object.
@@ -385,87 +416,317 @@ public final class TimelineStorageUtils {
}
/**
+ * Matches key-values filter. Used for relatesTo/isRelatedTo filters.
*
- * @param entityRelations the relations of an entity
- * @param relationFilters the relations for filtering
- * @return a boolean flag to indicate if both match
+ * @param entity entity which holds relatesTo/isRelatedTo relations which we
+ * will match against.
+ * @param keyValuesFilter key-values filter.
+ * @param entityFiltersType type of filters we are trying to match.
+ * @return true, if filter matches, false otherwise.
*/
- public static boolean matchRelations(
- Map<String, Set<String>> entityRelations,
- Map<String, Set<String>> relationFilters) {
- for (Map.Entry<String, Set<String>> relation : relationFilters.entrySet()) {
- Set<String> ids = entityRelations.get(relation.getKey());
- if (ids == null) {
+ private static boolean matchKeyValuesFilter(TimelineEntity entity,
+ TimelineKeyValuesFilter keyValuesFilter,
+ TimelineEntityFiltersType entityFiltersType) {
+ Map<String, Set<String>> relations = null;
+ if (entityFiltersType == TimelineEntityFiltersType.IS_RELATED_TO) {
+ relations = entity.getIsRelatedToEntities();
+ } else if (entityFiltersType == TimelineEntityFiltersType.RELATES_TO) {
+ relations = entity.getRelatesToEntities();
+ }
+ if (relations == null) {
+ return false;
+ }
+ Set<String> ids = relations.get(keyValuesFilter.getKey());
+ if (ids == null) {
+ return false;
+ }
+ boolean matched = false;
+ for (Object id : keyValuesFilter.getValues()) {
+ // Matches if id is found amongst the relationships for an entity and
+ // filter's compare op is EQUAL.
+ // If compare op is NOT_EQUAL, for a match to occur, id should not be
+ // found amongst relationships for an entity.
+ matched = !(ids.contains(id) ^
+ keyValuesFilter.getCompareOp() == TimelineCompareOp.EQUAL);
+ if (!matched) {
return false;
}
- for (String id : relation.getValue()) {
- if (!ids.contains(id)) {
- return false;
- }
- }
}
return true;
}
/**
+ * Matches relatesto.
*
- * @param map the map of key/value pairs in an entity
- * @param filters the map of key/value pairs for filtering
- * @return a boolean flag to indicate if both match
+ * @param entity entity which holds relatesto relations.
+ * @param relatesTo the relations for filtering.
+ * @return true, if filter matches, false otherwise.
+ * @throws IOException if an unsupported filter for matching relations is
+ * being matched.
*/
- public static boolean matchFilters(Map<String, ? extends Object> map,
- Map<String, ? extends Object> filters) {
- for (Map.Entry<String, ? extends Object> filter : filters.entrySet()) {
- Object value = map.get(filter.getKey());
- if (value == null) {
- return false;
- }
- if (!value.equals(filter.getValue())) {
- return false;
- }
+ public static boolean matchRelatesTo(TimelineEntity entity,
+ TimelineFilterList relatesTo) throws IOException {
+ return matchFilters(
+ entity, relatesTo, TimelineEntityFiltersType.RELATES_TO);
+ }
+
+ /**
+ * Matches isrelatedto.
+ *
+ * @param entity entity which holds isRelatedTo relations.
+ * @param isRelatedTo the relations for filtering.
+ * @return true, if filter matches, false otherwise.
+ * @throws IOException if an unsupported filter for matching relations is
+ * being matched.
+ */
+ public static boolean matchIsRelatedTo(TimelineEntity entity,
+ TimelineFilterList isRelatedTo) throws IOException {
+ return matchFilters(
+ entity, isRelatedTo, TimelineEntityFiltersType.IS_RELATED_TO);
+ }
+
+ /**
+ * Matches key-value filter. Used for config and info filters.
+ *
+ * @param entity entity which holds the config/info which we will match
+ * against.
+ * @param kvFilter a key-value filter.
+ * @param entityFiltersType type of filters we are trying to match.
+ * @return true, if filter matches, false otherwise.
+ */
+ private static boolean matchKeyValueFilter(TimelineEntity entity,
+ TimelineKeyValueFilter kvFilter,
+ TimelineEntityFiltersType entityFiltersType) {
+ Map<String, ? extends Object> map = null;
+ // Supported only for config and info filters.
+ if (entityFiltersType == TimelineEntityFiltersType.CONFIG) {
+ map = entity.getConfigs();
+ } else if (entityFiltersType == TimelineEntityFiltersType.INFO) {
+ map = entity.getInfo();
}
- return true;
+ if (map == null) {
+ return false;
+ }
+ Object value = map.get(kvFilter.getKey());
+ if (value == null) {
+ return false;
+ }
+ // Matches if filter's value is equal to the value of the key and filter's
+ // compare op is EQUAL.
+ // If compare op is NOT_EQUAL, for a match to occur, value should not be
+ // equal to the value of the key.
+ return !(value.equals(kvFilter.getValue()) ^
+ kvFilter.getCompareOp() == TimelineCompareOp.EQUAL);
+ }
+
+ /**
+ * Matches config filters.
+ *
+ * @param entity entity which holds a map of config key-value pairs.
+ * @param configFilters list of info filters.
+ * @return a boolean flag to indicate if both match.
+ * @throws IOException if an unsupported filter for matching config filters is
+ * being matched.
+ */
+ public static boolean matchConfigFilters(TimelineEntity entity,
+ TimelineFilterList configFilters) throws IOException {
+ return
+ matchFilters(entity, configFilters, TimelineEntityFiltersType.CONFIG);
+ }
+
+ /**
+ * Matches info filters.
+ *
+ * @param entity entity which holds a map of info key-value pairs.
+ * @param infoFilters list of info filters.
+ * @return a boolean flag to indicate if both match.
+ * @throws IOException if an unsupported filter for matching info filters is
+ * being matched.
+ */
+ public static boolean matchInfoFilters(TimelineEntity entity,
+ TimelineFilterList infoFilters) throws IOException {
+ return matchFilters(entity, infoFilters, TimelineEntityFiltersType.INFO);
}
/**
+ * Matches exists filter. Used for event filters.
*
- * @param entityEvents the set of event objects in an entity
- * @param eventFilters the set of event Ids for filtering
- * @return a boolean flag to indicate if both match
+ * @param entity entity which holds the events which we will match against.
+ * @param existsFilter exists filter.
+ * @param entityFiltersType type of filters we are trying to match.
+ * @return true, if filter matches, false otherwise.
*/
- public static boolean matchEventFilters(Set<TimelineEvent> entityEvents,
- Set<String> eventFilters) {
+ private static boolean matchExistsFilter(TimelineEntity entity,
+ TimelineExistsFilter existsFilter,
+ TimelineEntityFiltersType entityFiltersType) {
+ // Currently exists filter is only supported for event filters.
+ if (entityFiltersType != TimelineEntityFiltersType.EVENT) {
+ return false;
+ }
Set<String> eventIds = new HashSet<String>();
- for (TimelineEvent event : entityEvents) {
+ for (TimelineEvent event : entity.getEvents()) {
eventIds.add(event.getId());
}
- for (String eventFilter : eventFilters) {
- if (!eventIds.contains(eventFilter)) {
- return false;
- }
+ // Matches if filter's value is contained in the list of events filter's
+ // compare op is EQUAL.
+ // If compare op is NOT_EQUAL, for a match to occur, value should not be
+ // contained in the list of events.
+ return !(eventIds.contains(existsFilter.getValue()) ^
+ existsFilter.getCompareOp() == TimelineCompareOp.EQUAL);
+ }
+
+ /**
+ * Matches event filters.
+ *
+ * @param entity entity which holds a set of event objects.
+ * @param eventFilters the set of event Ids for filtering.
+ * @return a boolean flag to indicate if both match.
+ * @throws IOException if an unsupported filter for matching event filters is
+ * being matched.
+ */
+ public static boolean matchEventFilters(TimelineEntity entity,
+ TimelineFilterList eventFilters) throws IOException {
+ return matchFilters(entity, eventFilters, TimelineEntityFiltersType.EVENT);
+ }
+
+ /**
+ * Compare two values based on comparison operator.
+ *
+ * @param compareOp comparison operator.
+ * @param val1 value 1.
+ * @param val2 value 2.
+ * @return true, if relation matches, false otherwise
+ */
+ private static boolean compareValues(TimelineCompareOp compareOp,
+ long val1, long val2) {
+ switch (compareOp) {
+ case LESS_THAN:
+ return val1 < val2;
+ case LESS_OR_EQUAL:
+ return val1 <= val2;
+ case EQUAL:
+ return val1 == val2;
+ case NOT_EQUAL:
+ return val1 != val2;
+ case GREATER_OR_EQUAL:
+ return val1 >= val2;
+ case GREATER_THAN:
+ return val1 > val2;
+ default:
+ throw new RuntimeException("Unknown TimelineCompareOp " +
+ compareOp.name());
}
- return true;
}
/**
+ * Matches compare filter. Used for metric filters.
*
- * @param metrics the set of metric objects in an entity
- * @param metricFilters the set of metric Ids for filtering
- * @return a boolean flag to indicate if both match
+ * @param entity entity which holds the metrics which we will match against.
+ * @param compareFilter compare filter.
+ * @param entityFiltersType type of filters we are trying to match.
+ * @return true, if filter matches, false otherwise.
+ * @throws IOException if metric filters holds non integral values.
*/
- public static boolean matchMetricFilters(Set<TimelineMetric> metrics,
- Set<String> metricFilters) {
- Set<String> metricIds = new HashSet<String>();
- for (TimelineMetric metric : metrics) {
- metricIds.add(metric.getId());
+ private static boolean matchCompareFilter(TimelineEntity entity,
+ TimelineCompareFilter compareFilter,
+ TimelineEntityFiltersType entityFiltersType) throws IOException {
+ // Currently exists filter is only supported for metric filters.
+ if (entityFiltersType != TimelineEntityFiltersType.METRIC) {
+ return false;
+ }
+ // We expect only integral values(short/int/long) for metric filters.
+ if (!isIntegralValue(compareFilter.getValue())) {
+ throw new IOException("Metric filters has non integral values");
+ }
+ Map<String, TimelineMetric> metricMap =
+ new HashMap<String, TimelineMetric>();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ metricMap.put(metric.getId(), metric);
}
+ TimelineMetric metric = metricMap.get(compareFilter.getKey());
+ if (metric == null) {
+ return false;
+ }
+ // We will be using the latest value of metric to compare.
+ return compareValues(compareFilter.getCompareOp(),
+ metric.getValuesJAXB().firstEntry().getValue().longValue(),
+ ((Number)compareFilter.getValue()).longValue());
+ }
- for (String metricFilter : metricFilters) {
- if (!metricIds.contains(metricFilter)) {
- return false;
+ /**
+ * Matches metric filters.
+ *
+ * @param entity entity which holds a set of metric objects.
+ * @param metricFilters list of metric filters.
+ * @return a boolean flag to indicate if both match.
+ * @throws IOException if an unsupported filter for matching metric filters is
+ * being matched.
+ */
+ public static boolean matchMetricFilters(TimelineEntity entity,
+ TimelineFilterList metricFilters) throws IOException {
+ return matchFilters(
+ entity, metricFilters, TimelineEntityFiltersType.METRIC);
+ }
+
+ /**
+ * Common routine to match different filters. Iterates over a filter list and
+ * calls routines based on filter type.
+ *
+ * @param entity Timeline entity.
+ * @param filters filter list.
+ * @param entityFiltersType type of filters which are being matched.
+ * @return a boolean flag to indicate if filter matches.
+ * @throws IOException if an unsupported filter for matching this specific
+ * filter is being matched.
+ */
+ private static boolean matchFilters(TimelineEntity entity,
+ TimelineFilterList filters, TimelineEntityFiltersType entityFiltersType)
+ throws IOException {
+ if (filters == null || filters.getFilterList().isEmpty()) {
+ return false;
+ }
+ TimelineFilterList.Operator operator = filters.getOperator();
+ for (TimelineFilter filter : filters.getFilterList()) {
+ TimelineFilterType filterType = filter.getFilterType();
+ if (!entityFiltersType.isValidFilter(filterType)) {
+ throw new IOException("Unsupported filter " + filterType);
+ }
+ boolean matched = false;
+ switch (filterType) {
+ case LIST:
+ matched = matchFilters(entity, (TimelineFilterList)filter,
+ entityFiltersType);
+ break;
+ case COMPARE:
+ matched = matchCompareFilter(entity, (TimelineCompareFilter)filter,
+ entityFiltersType);
+ break;
+ case EXISTS:
+ matched = matchExistsFilter(entity, (TimelineExistsFilter)filter,
+ entityFiltersType);
+ break;
+ case KEY_VALUE:
+ matched = matchKeyValueFilter(entity, (TimelineKeyValueFilter)filter,
+ entityFiltersType);
+ break;
+ case KEY_VALUES:
+ matched = matchKeyValuesFilter(entity, (TimelineKeyValuesFilter)filter,
+ entityFiltersType);
+ break;
+ default:
+ throw new IOException("Unsupported filter " + filterType);
+ }
+ if (!matched) {
+ if(operator == TimelineFilterList.Operator.AND) {
+ return false;
+ }
+ } else {
+ if(operator == TimelineFilterList.Operator.OR) {
+ return true;
+ }
}
}
- return true;
+ return operator == TimelineFilterList.Operator.AND;
}
/**
@@ -530,4 +791,100 @@ public final class TimelineStorageUtils {
}
return appId;
}
+
+ /**
+ * Helper method for reading relationship.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result result from HBase.
+ * @param prefix column prefix.
+ * @param isRelatedTo if true, means relationship is to be added to
+ * isRelatedTo, otherwise its added to relatesTo.
+ * @throws IOException if any problem is encountered while reading result.
+ */
+ public static <T> void readRelationship(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isRelatedTo) throws IOException {
+ // isRelatedTo and relatesTo are of type Map<String, Set<String>>
+ Map<String, Object> columns = prefix.readResults(result);
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ for (String id : Separator.VALUES.splitEncoded(
+ column.getValue().toString())) {
+ if (isRelatedTo) {
+ entity.addIsRelatedToEntity(column.getKey(), id);
+ } else {
+ entity.addRelatesToEntity(column.getKey(), id);
+ }
+ }
+ }
+ }
+
+ /**
+ * Helper method for reading key-value pairs for either info or config.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result result from HBase.
+ * @param prefix column prefix.
+ * @param isConfig if true, means we are reading configs, otherwise info.
+ * @throws IOException if any problem is encountered while reading result.
+ */
+ public static <T> void readKeyValuePairs(
+ TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
+ boolean isConfig) throws IOException {
+ // info and configuration are of type Map<String, Object or String>
+ Map<String, Object> columns = prefix.readResults(result);
+ if (isConfig) {
+ for (Map.Entry<String, Object> column : columns.entrySet()) {
+ entity.addConfig(column.getKey(), column.getValue().toString());
+ }
+ } else {
+ entity.addInfo(columns);
+ }
+ }
+
+ /**
+ * Read events from the entity table or the application table. The column name
+ * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
+ * if there is no info associated with the event.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result HBase Result.
+ * @param prefix column prefix.
+ * @throws IOException if any problem is encountered while reading result.
+ */
+ public static <T> void readEvents(TimelineEntity entity, Result result,
+ ColumnPrefix<T> prefix) throws IOException {
+ Map<String, TimelineEvent> eventsMap = new HashMap<>();
+ Map<?, Object> eventsResult =
+ prefix.readResultsHavingCompoundColumnQualifiers(result);
+ for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
+ byte[][] karr = (byte[][])eventResult.getKey();
+ // the column name is of the form "eventId=timestamp=infoKey"
+ if (karr.length == 3) {
+ String id = Bytes.toString(karr[0]);
+ long ts = TimelineStorageUtils.invertLong(Bytes.toLong(karr[1]));
+ String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
+ TimelineEvent event = eventsMap.get(key);
+ if (event == null) {
+ event = new TimelineEvent();
+ event.setId(id);
+ event.setTimestamp(ts);
+ eventsMap.put(key, event);
+ }
+ // handle empty info
+ String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
+ if (infoKey != null) {
+ event.addInfo(infoKey, eventResult.getValue());
+ }
+ } else {
+ LOG.warn("incorrectly formatted column name: it will be discarded");
+ continue;
+ }
+ }
+ Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
+ entity.addEvents(eventsSet);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index f47ba93..775879a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -24,8 +24,11 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Column;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
/**
@@ -46,7 +49,8 @@ public enum EntityColumn implements Column<EntityTable> {
/**
* When the entity was created.
*/
- CREATED_TIME(EntityColumnFamily.INFO, "created_time"),
+ CREATED_TIME(EntityColumnFamily.INFO, "created_time",
+ LongConverter.getInstance()),
/**
* The version of the flow that this entity belongs to.
@@ -60,12 +64,17 @@ public enum EntityColumn implements Column<EntityTable> {
EntityColumn(ColumnFamily<EntityTable> columnFamily,
String columnQualifier) {
+ this(columnFamily, columnQualifier, GenericConverter.getInstance());
+ }
+
+ EntityColumn(ColumnFamily<EntityTable> columnFamily,
+ String columnQualifier, ValueConverter converter) {
this.columnFamily = columnFamily;
this.columnQualifier = columnQualifier;
// Future-proof by ensuring the right column prefix hygiene.
this.columnQualifierBytes =
Bytes.toBytes(Separator.SPACE.encode(columnQualifier));
- this.column = new ColumnHelper<EntityTable>(columnFamily);
+ this.column = new ColumnHelper<EntityTable>(columnFamily, converter);
}
/**
@@ -108,6 +117,21 @@ public enum EntityColumn implements Column<EntityTable> {
return null;
}
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return columnQualifierBytes.clone();
+ }
+
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return column.getValueConverter();
+ }
+
/**
* Retrieve an {@link EntityColumn} given a name, or null if there is no
* match. The following holds true: {@code columnFor(a,x) == columnFor(b,y)}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index f3c7e7f..de2b29d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -56,7 +56,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
/**
* Lifecycle events for an entity.
*/
- EVENT(EntityColumnFamily.INFO, "e"),
+ EVENT(EntityColumnFamily.INFO, "e", true),
/**
* Config column stores configuration with config key as the column name.
@@ -78,6 +78,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*/
private final String columnPrefix;
private final byte[] columnPrefixBytes;
+ private final boolean compoundColQual;
/**
* Private constructor, meant to be used by the enum definition.
@@ -87,7 +88,18 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
*/
EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
String columnPrefix) {
- this(columnFamily, columnPrefix, GenericConverter.getInstance());
+ this(columnFamily, columnPrefix, false, GenericConverter.getInstance());
+ }
+
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix, boolean compondColQual) {
+ this(columnFamily, columnPrefix, compondColQual,
+ GenericConverter.getInstance());
+ }
+
+ EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
+ String columnPrefix, ValueConverter converter) {
+ this(columnFamily, columnPrefix, false, converter);
}
/**
@@ -99,7 +111,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* this column prefix.
*/
EntityColumnPrefix(ColumnFamily<EntityTable> columnFamily,
- String columnPrefix, ValueConverter converter) {
+ String columnPrefix, boolean compondColQual, ValueConverter converter) {
column = new ColumnHelper<EntityTable>(columnFamily, converter);
this.columnFamily = columnFamily;
this.columnPrefix = columnPrefix;
@@ -110,6 +122,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
this.columnPrefixBytes =
Bytes.toBytes(Separator.SPACE.encode(columnPrefix));
}
+ this.compoundColQual = compondColQual;
}
/**
@@ -131,6 +144,24 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
this.columnPrefixBytes, qualifierPrefix);
}
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return column.getValueConverter();
+ }
+
+ public byte[] getCompoundColQualBytes(String qualifier,
+ byte[]...components) {
+ if (!compoundColQual) {
+ return ColumnHelper.getColumnQualifier(null, qualifier);
+ }
+ return ColumnHelper.getCompoundColumnQualifierBytes(qualifier, components);
+ }
+
/*
* (non-Javadoc)
*
@@ -287,5 +318,4 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
// Default to null
return null;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
index a5933da..188c2fe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
/**
* Identifies partially qualified columns for the {@link FlowActivityTable}.
@@ -50,6 +51,7 @@ public enum FlowActivityColumnPrefix
*/
private final String columnPrefix;
private final byte[] columnPrefixBytes;
+ private final boolean compoundColQual;
private final AggregationOperation aggOp;
@@ -64,6 +66,12 @@ public enum FlowActivityColumnPrefix
private FlowActivityColumnPrefix(
ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
AggregationOperation aggOp) {
+ this(columnFamily, columnPrefix, aggOp, false);
+ }
+
+ private FlowActivityColumnPrefix(
+ ColumnFamily<FlowActivityTable> columnFamily, String columnPrefix,
+ AggregationOperation aggOp, boolean compoundColQual) {
column = new ColumnHelper<FlowActivityTable>(columnFamily);
this.columnFamily = columnFamily;
this.columnPrefix = columnPrefix;
@@ -75,6 +83,7 @@ public enum FlowActivityColumnPrefix
.encode(columnPrefix));
}
this.aggOp = aggOp;
+ this.compoundColQual = compoundColQual;
}
/**
@@ -100,6 +109,16 @@ public enum FlowActivityColumnPrefix
return columnPrefixBytes.clone();
}
+ @Override
+ public byte[] getColumnFamilyBytes() {
+ return columnFamily.getBytes();
+ }
+
+ @Override
+ public ValueConverter getValueConverter() {
+ return column.getValueConverter();
+ }
+
public AggregationOperation getAttribute() {
return aggOp;
}
@@ -251,4 +270,20 @@ public enum FlowActivityColumnPrefix
column.store(rowKey, tableMutator, columnQualifier, null, inputValue,
combinedAttributes);
}
+
+ @Override
+ public byte[] getCompoundColQualBytes(String qualifier,
+ byte[]...components) {
+ if (!compoundColQual) {
+ return ColumnHelper.getColumnQualifier(null, qualifier);
+ }
+ return ColumnHelper.getCompoundColumnQualifierBytes(qualifier, components);
+ }
+
+ @Override
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException {
+ // There are no compound column qualifiers for flow activity table.
+ return null;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
index d50bb16..f1553b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
@@ -86,10 +86,12 @@ public enum FlowRunColumn implements Column<FlowRunTable> {
return columnQualifier;
}
+ @Override
public byte[] getColumnQualifierBytes() {
return columnQualifierBytes.clone();
}
+ @Override
public byte[] getColumnFamilyBytes() {
return columnFamily.getBytes();
}
@@ -144,6 +146,7 @@ public enum FlowRunColumn implements Column<FlowRunTable> {
return null;
}
+ @Override
public ValueConverter getValueConverter() {
return column.getValueConverter();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
index fa94aae..77f2ab2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
@@ -52,6 +52,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
*/
private final String columnPrefix;
private final byte[] columnPrefixBytes;
+ private final boolean compoundColQual;
private final AggregationOperation aggOp;
@@ -65,6 +66,12 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
*/
private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
String columnPrefix, AggregationOperation fra, ValueConverter converter) {
+ this(columnFamily, columnPrefix, fra, converter, false);
+ }
+
+ private FlowRunColumnPrefix(ColumnFamily<FlowRunTable> columnFamily,
+ String columnPrefix, AggregationOperation fra, ValueConverter converter,
+ boolean compoundColQual) {
column = new ColumnHelper<FlowRunTable>(columnFamily, converter);
this.columnFamily = columnFamily;
this.columnPrefix = columnPrefix;
@@ -76,6 +83,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
.encode(columnPrefix));
}
this.aggOp = fra;
+ this.compoundColQual = compoundColQual;
}
/**
@@ -101,6 +109,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
this.columnPrefixBytes, qualifierPrefix);
}
+ @Override
public byte[] getColumnFamilyBytes() {
return columnFamily.getBytes();
}
@@ -222,6 +231,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
return null;
}
+ @Override
public ValueConverter getValueConverter() {
return column.getValueConverter();
}
@@ -257,4 +267,20 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
// Default to null
return null;
}
+
+ @Override
+ public byte[] getCompoundColQualBytes(String qualifier,
+ byte[]...components) {
+ if (!compoundColQual) {
+ return ColumnHelper.getColumnQualifier(null, qualifier);
+ }
+ return ColumnHelper.getCompoundColumnQualifierBytes(qualifier, components);
+ }
+
+ @Override
+ public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
+ throws IOException {
+ // There are no compound column qualifiers for flow run table.
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
index 6baea37..0ace529 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -200,6 +200,7 @@ class FlowScanner implements RegionScanner, Closeable {
int addedCnt = 0;
long currentTimestamp = System.currentTimeMillis();
ValueConverter converter = null;
+
while (cellLimit <= 0 || addedCnt < cellLimit) {
cell = peekAtNextCell(cellLimit);
if (cell == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index 0de09e0..53210f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
import java.util.EnumSet;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
@@ -27,7 +28,6 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.FilterList;
@@ -39,6 +39,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
@@ -76,93 +77,231 @@ class ApplicationEntityReader extends GenericEntityReader {
return APPLICATION_TABLE;
}
+ /**
+ * This method is called only for multiple entity reads.
+ */
@Override
- protected FilterList constructFilterListBasedOnFields() {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Fetch all the columns.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (dataToRetrieve.getConfsToRetrieve() == null ||
- dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) &&
- (dataToRetrieve.getMetricsToRetrieve() == null ||
- dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
- return list;
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ // Filters here cannot be null for multiple entity reads as they are set in
+ // augmentParams if null.
+ TimelineEntityFilters filters = getFilters();
+ FilterList listBasedOnFilters = new FilterList();
+ // Create filter list based on created time range and add it to
+ // listBasedOnFilters.
+ long createdTimeBegin = filters.getCreatedTimeBegin();
+ long createdTimeEnd = filters.getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createSingleColValueFiltersByRange(
+ ApplicationColumn.CREATED_TIME, createdTimeBegin, createdTimeEnd));
}
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
+ // Create filter list based on metric filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList metricFilters = filters.getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.METRIC, metricFilters));
+ }
+ // Create filter list based on config filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList configFilters = filters.getConfigFilters();
+ if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.CONFIG, configFilters));
+ }
+ // Create filter list based on info filters and add it to listBasedOnFilters
+ TimelineFilterList infoFilters = filters.getInfoFilters();
+ if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ ApplicationColumnPrefix.INFO, infoFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of
+ * application table.
+ *
+ * @param list filter list to which qualifier filters have to be added.
+ */
+ @Override
+ protected void updateFixedColumns(FilterList list) {
+ for (ApplicationColumn column : ApplicationColumn.values()) {
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryComparator(column.getColumnQualifierBytes())));
+ }
+ }
+
+ /**
+ * Creates a filter list which indicates that only some of the column
+ * qualifiers in the info column family will be returned in result.
+ *
+ * @return filter list.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterListForColsOfInfoFamily()
+ throws IOException {
+ FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
+ // Add filters for each column in entity table.
+ updateFixedColumns(infoFamilyColsFilter);
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // If INFO field has to be retrieved, add a filter for fetching columns
+ // with INFO column prefix.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.INFO)) {
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.INFO));
+ }
+ TimelineFilterList relatesTo = getFilters().getRelatesTo();
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ // If RELATES_TO field has to be retrieved, add a filter for fetching
+ // columns with RELATES_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.RELATES_TO));
+ } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain RELATES_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // relatesTo filters are specified. relatesTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> relatesToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createFiltersFromColumnQualifiers(
+ ApplicationColumnPrefix.RELATES_TO, relatesToCols));
+ }
+ TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
+ // columns with IS_RELATED_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.IS_RELATED_TO));
+ } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain IS_RELATED_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // isRelatedTo filters are specified. isRelatedTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> isRelatedToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createFiltersFromColumnQualifiers(
+ ApplicationColumnPrefix.IS_RELATED_TO, isRelatedToCols));
+ }
+ TimelineFilterList eventFilters = getFilters().getEventFilters();
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS)) {
+ // If EVENTS field has to be retrieved, add a filter for fetching columns
+ // with EVENT column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, ApplicationColumnPrefix.EVENT));
+ } else if (eventFilters != null && !eventFilters.getFilterList().isEmpty()){
+ // Even if fields to retrieve does not contain EVENTS, we still need to
+ // have a filter to fetch some of the column qualifiers on the basis of
+ // event filters specified. Event filters will then be matched after
+ // fetching rows from HBase.
+ Set<String> eventCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createFiltersFromColumnQualifiers(
+ ApplicationColumnPrefix.EVENT, eventCols));
+ }
+ return infoFamilyColsFilter;
+ }
+
+ /**
+ * Exclude column prefixes via filters which are not required(based on fields
+ * to retrieve) from info column family. These filters are added to filter
+ * list which contains a filter for getting info column family.
+ *
+ * @param infoColFamilyList filter list for info column family.
+ */
+ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
// Events not required.
- TimelineEntityFilters filters = getFilters();
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.EVENTS) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getEventFilters() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.EVENT.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.EVENT));
}
// info not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.INFO) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getInfoFilters() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.INFO)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.INFO.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.INFO));
}
- // is releated to not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.IS_RELATED_TO) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getIsRelatedTo() == null)) {
+ // is related to not required.
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.IS_RELATED_TO));
}
// relates to not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.RELATES_TO) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getRelatesTo() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, ApplicationColumnPrefix.RELATES_TO));
}
- list.addFilter(infoColFamilyList);
- if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS) ||
- (!isSingleEntityRead() && filters.getConfigFilters() != null)) ||
- (dataToRetrieve.getConfsToRetrieve() != null &&
- !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty())) {
- FilterList filterCfg =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.CONFIGS.getBytes())));
- if (dataToRetrieve.getConfsToRetrieve() != null &&
- !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) {
- filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.CONFIG,
- dataToRetrieve.getConfsToRetrieve()));
- }
- list.addFilter(filterCfg);
+ }
+
+ /**
+ * Updates filter list based on fields for confs and metrics to retrieve.
+ *
+ * @param listBasedOnFields filter list based on fields.
+ * @throws IOException if any problem occurs while updating filter list.
+ */
+ private void updateFilterForConfsAndMetricsToRetrieve(
+ FilterList listBasedOnFields) throws IOException {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Please note that if confsToRetrieve is specified, we would have added
+ // CONFS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
+ // Create a filter list for configs.
+ listBasedOnFields.addFilter(TimelineFilterUtils.
+ createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getConfsToRetrieve(),
+ ApplicationColumnFamily.CONFIGS, ApplicationColumnPrefix.CONFIG));
}
- if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) ||
- (!isSingleEntityRead() && filters.getMetricFilters() != null)) ||
- (dataToRetrieve.getMetricsToRetrieve() != null &&
- !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
- FilterList filterMetrics =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.METRICS.getBytes())));
- if (dataToRetrieve.getMetricsToRetrieve() != null &&
- !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty()) {
- filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.METRIC,
- dataToRetrieve.getMetricsToRetrieve()));
- }
- list.addFilter(filterMetrics);
+
+ // Please note that if metricsToRetrieve is specified, we would have added
+ // METRICS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
+ // Create a filter list for metrics.
+ listBasedOnFields.addFilter(TimelineFilterUtils.
+ createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getMetricsToRetrieve(),
+ ApplicationColumnFamily.METRICS, ApplicationColumnPrefix.METRIC));
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
+ if (!needCreateFilterListBasedOnFields()) {
+ // Fetch all the columns. No need of a filter.
+ return null;
+ }
+ FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ if (!isSingleEntityRead() && fetchPartialColsFromInfoFamily()) {
+ // We can fetch only some of the columns from info family.
+ infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
+ } else {
+ // Exclude column prefixes in info column family which are not required
+ // based on fields to retrieve.
+ excludeFieldsFromInfoColFamily(infoColFamilyList);
}
- return list;
+ listBasedOnFields.addFilter(infoColFamilyList);
+
+ updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
+ return listBasedOnFields;
}
@Override
@@ -182,6 +321,9 @@ class ApplicationEntityReader extends GenericEntityReader {
@Override
protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(
+ getDataToRetrieve(), "data to retrieve shouldn't be null");
Preconditions.checkNotNull(getContext().getClusterId(),
"clusterId shouldn't be null");
Preconditions.checkNotNull(getContext().getEntityType(),
@@ -202,6 +344,7 @@ class ApplicationEntityReader extends GenericEntityReader {
throws IOException {
TimelineReaderContext context = getContext();
if (isSingleEntityRead()) {
+ // Get flow context information from AppToFlow table.
if (context.getFlowName() == null || context.getFlowRunId() == null ||
context.getUserId() == null) {
FlowContext flowContext = lookupFlowContext(
@@ -211,7 +354,12 @@ class ApplicationEntityReader extends GenericEntityReader {
context.setUserId(flowContext.getUserId());
}
}
+ // Add configs/metrics to fields to retrieve if confsToRetrieve and/or
+ // metricsToRetrieve are specified.
getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ if (!isSingleEntityRead()) {
+ createFiltersIfNull();
+ }
}
@Override
@@ -252,114 +400,84 @@ class ApplicationEntityReader extends GenericEntityReader {
Number createdTime =
(Number)ApplicationColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue());
- if (!isSingleEntityRead() &&
- (entity.getCreatedTime() < filters.getCreatedTimeBegin() ||
- entity.getCreatedTime() > filters.getCreatedTimeEnd())) {
- return null;
- }
+
EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // fetch is related to entities
+ // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // isRelatedTo are not set in HBase scan.
boolean checkIsRelatedTo =
- filters != null && filters.getIsRelatedTo() != null &&
- filters.getIsRelatedTo().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
- true);
- if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) {
+ !isSingleEntityRead() && filters.getIsRelatedTo() != null &&
+ filters.getIsRelatedTo().getFilterList().size() > 0;
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO) ||
+ checkIsRelatedTo) {
+ TimelineStorageUtils.readRelationship(
+ entity, result, ApplicationColumnPrefix.IS_RELATED_TO, true);
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity,
+ filters.getIsRelatedTo())) {
return null;
}
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve,
+ Field.IS_RELATED_TO)) {
entity.getIsRelatedToEntities().clear();
}
}
- // fetch relates to entities
+ // fetch relates to entities and match relatesTo filter. If relatesTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // relatesTo are not set in HBase scan.
boolean checkRelatesTo =
- filters != null && filters.getRelatesTo() != null &&
- filters.getRelatesTo().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
- false);
- if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), filters.getRelatesTo())) {
+ !isSingleEntityRead() && filters.getRelatesTo() != null &&
+ filters.getRelatesTo().getFilterList().size() > 0;
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO) ||
+ checkRelatesTo) {
+ TimelineStorageUtils.readRelationship(
+ entity, result, ApplicationColumnPrefix.RELATES_TO, false);
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity,
+ filters.getRelatesTo())) {
return null;
}
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO)) {
entity.getRelatesToEntities().clear();
}
}
- // fetch info
- boolean checkInfo = filters != null && filters.getInfoFilters() != null &&
- filters.getInfoFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
- if (checkInfo &&
- !TimelineStorageUtils.matchFilters(
- entity.getInfo(), filters.getInfoFilters())) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.INFO)) {
- entity.getInfo().clear();
- }
+ // fetch info if fieldsToRetrieve contains INFO or ALL.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.INFO)) {
+ TimelineStorageUtils.readKeyValuePairs(
+ entity, result, ApplicationColumnPrefix.INFO, false);
}
- // fetch configs
- boolean checkConfigs =
- filters != null && filters.getConfigFilters() != null &&
- filters.getConfigFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
- if (checkConfigs && !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), filters.getConfigFilters())) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.CONFIGS)) {
- entity.getConfigs().clear();
- }
+ // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.CONFIGS)) {
+ TimelineStorageUtils.readKeyValuePairs(
+ entity, result, ApplicationColumnPrefix.CONFIG, true);
}
- // fetch events
+ // fetch events and match event filters if they exist. If event filters do
+ // not match, entity would be dropped. We have to match filters locally
+ // as relevant HBase filters to filter out rows on the basis of events
+ // are not set in HBase scan.
boolean checkEvents =
- filters != null && filters.getEventFilters() != null &&
- filters.getEventFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result, true);
- if (checkEvents && !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), filters.getEventFilters())) {
+ !isSingleEntityRead() && filters.getEventFilters() != null &&
+ filters.getEventFilters().getFilterList().size() > 0;
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS) ||
+ checkEvents) {
+ TimelineStorageUtils.readEvents(
+ entity, result, ApplicationColumnPrefix.EVENT);
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity,
+ filters.getEventFilters())) {
return null;
}
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.EVENTS)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS)) {
entity.getEvents().clear();
}
}
- // fetch metrics
- boolean checkMetrics =
- filters != null && filters.getMetricFilters() != null &&
- filters.getMetricFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.METRICS)) {
readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
- if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), filters.getMetricFilters())) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.METRICS)) {
- entity.getMetrics().clear();
- }
}
return entity;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
index 0d2bdd8..d8ca038 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -15,6 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
@@ -75,6 +76,12 @@ class FlowActivityEntityReader extends TimelineEntityReader {
@Override
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
+ createFiltersIfNull();
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
index 743315c..b2de2e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -38,9 +38,11 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
@@ -76,6 +78,9 @@ class FlowRunEntityReader extends TimelineEntityReader {
@Override
protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(
+ getDataToRetrieve(), "data to retrieve shouldn't be null");
Preconditions.checkNotNull(getContext().getClusterId(),
"clusterId shouldn't be null");
Preconditions.checkNotNull(getContext().getUserId(),
@@ -90,37 +95,87 @@ class FlowRunEntityReader extends TimelineEntityReader {
@Override
protected void augmentParams(Configuration hbaseConf, Connection conn) {
+ // Add metrics to fields to retrieve if metricsToRetrieve is specified.
getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ if (!isSingleEntityRead()) {
+ createFiltersIfNull();
+ }
+ }
+
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ FilterList listBasedOnFilters = new FilterList();
+ // Filter based on created time range.
+ Long createdTimeBegin = getFilters().getCreatedTimeBegin();
+ Long createdTimeEnd = getFilters().getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createSingleColValueFiltersByRange(
+ FlowRunColumn.MIN_START_TIME, createdTimeBegin, createdTimeEnd));
+ }
+ // Filter based on metric filters.
+ TimelineFilterList metricFilters = getFilters().getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ FlowRunColumnPrefix.METRIC, metricFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of
+ * flow run table.
+ *
+ * @return filter list to which qualifier filters have been added.
+ */
+ private FilterList updateFixedColumns() {
+ FilterList columnsList = new FilterList(Operator.MUST_PASS_ONE);
+ for (FlowRunColumn column : FlowRunColumn.values()) {
+ columnsList.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryComparator(column.getColumnQualifierBytes())));
+ }
+ return columnsList;
}
@Override
- protected FilterList constructFilterListBasedOnFields() {
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
FilterList list = new FilterList(Operator.MUST_PASS_ONE);
-
// By default fetch everything in INFO column family.
FamilyFilter infoColumnFamily =
new FamilyFilter(CompareOp.EQUAL,
new BinaryComparator(FlowRunColumnFamily.INFO.getBytes()));
TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Metrics not required.
- if (!isSingleEntityRead() &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL)) {
+ // If multiple entities have to be retrieved, check if metrics have to be
+ // retrieved and if not, add a filter so that metrics can be excluded.
+ // Metrics are always returned if we are reading a single entity.
+ if (!isSingleEntityRead() && !TimelineStorageUtils.hasField(
+ dataToRetrieve.getFieldsToRetrieve(), Field.METRICS)) {
FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
infoColFamilyList.addFilter(infoColumnFamily);
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
- FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(""))));
- list.addFilter(infoColFamilyList);
- }
- if (dataToRetrieve.getMetricsToRetrieve() != null &&
- !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty()) {
- FilterList infoColFamilyList = new FilterList();
- infoColFamilyList.addFilter(infoColumnFamily);
- infoColFamilyList.addFilter(TimelineFilterUtils.createHBaseFilterList(
- FlowRunColumnPrefix.METRIC, dataToRetrieve.getMetricsToRetrieve()));
+ FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(""))));
list.addFilter(infoColFamilyList);
+ } else {
+ // Check if metricsToRetrieve are specified and if they are, create a
+ // filter list for info column family by adding flow run tables columns
+ // and a list for metrics to retrieve. Pls note that fieldsToRetrieve
+ // will have METRICS added to it if metricsToRetrieve are specified
+ // (in augmentParams()).
+ TimelineFilterList metricsToRetrieve =
+ dataToRetrieve.getMetricsToRetrieve();
+ if (metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ FilterList infoColFamilyList = new FilterList();
+ infoColFamilyList.addFilter(infoColumnFamily);
+ FilterList columnsList = updateFixedColumns();
+ columnsList.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ FlowRunColumnPrefix.METRIC, metricsToRetrieve));
+ infoColFamilyList.addFilter(columnsList);
+ list.addFilter(infoColFamilyList);
+ }
}
return list;
}
@@ -175,11 +230,6 @@ class FlowRunEntityReader extends TimelineEntityReader {
if (startTime != null) {
flowRun.setStartTime(startTime.longValue());
}
- if (!isSingleEntityRead() &&
- (flowRun.getStartTime() < getFilters().getCreatedTimeBegin() ||
- flowRun.getStartTime() > getFilters().getCreatedTimeEnd())) {
- return null;
- }
// read the end time if available
Long endTime = (Long)FlowRunColumn.MAX_END_TIME.readResult(result);
@@ -193,9 +243,10 @@ class FlowRunEntityReader extends TimelineEntityReader {
flowRun.setVersion(version);
}
- // read metrics
- if (isSingleEntityRead() ||
- getDataToRetrieve().getFieldsToRetrieve().contains(Field.METRICS)) {
+ // read metrics if its a single entity query or if METRICS are part of
+ // fieldsToRetrieve.
+ if (isSingleEntityRead() || TimelineStorageUtils.hasField(
+ getDataToRetrieve().getFieldsToRetrieve(), Field.METRICS)) {
readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and
checkstyle issues in timelineservice code (Varun Saxena via sjlee)
Posted by gt...@apache.org.
YARN-4409. Fix javadoc and checkstyle issues in timelineservice code (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee803e16
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee803e16
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee803e16
Branch: refs/heads/YARN-2928
Commit: ee803e16e13bd01520e111dd448123c76d778225
Parents: 675612d
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Feb 8 12:17:43 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:28:05 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 170 ++++++++++---------
.../hadoop/mapreduce/jobhistory/TestEvents.java | 2 +-
.../mapreduce/util/JobHistoryEventUtils.java | 7 +-
.../hadoop/mapred/TimelineEntityConverter.java | 6 +-
.../hadoop/mapreduce/JobHistoryFileParser.java | 3 +
.../ApplicationAttemptEntity.java | 9 +-
.../timelineservice/ApplicationEntity.java | 9 +-
.../records/timelineservice/ClusterEntity.java | 6 +-
.../timelineservice/ContainerEntity.java | 9 +-
.../records/timelineservice/FlowRunEntity.java | 9 +-
.../HierarchicalTimelineEntity.java | 8 +-
.../records/timelineservice/QueueEntity.java | 6 +-
.../timelineservice/TimelineEntities.java | 11 +-
.../records/timelineservice/TimelineEntity.java | 106 ++++++------
.../timelineservice/TimelineEntityType.java | 71 +++++---
.../records/timelineservice/TimelineEvent.java | 30 ++--
.../records/timelineservice/TimelineMetric.java | 39 +++--
.../timelineservice/TimelineWriteResponse.java | 59 +++----
.../api/records/timelineservice/UserEntity.java | 6 +-
.../records/timelineservice/package-info.java | 8 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +-
.../hadoop/yarn/client/api/TimelineClient.java | 13 +-
.../client/api/impl/TimelineClientImpl.java | 16 +-
.../yarn/util/timeline/TimelineUtils.java | 21 ++-
.../yarn/server/nodemanager/NodeManager.java | 2 +-
.../collectormanager/NMCollectorService.java | 9 +-
.../collectormanager/package-info.java | 28 +++
.../timelineservice/NMTimelineEvent.java | 4 +
.../timelineservice/NMTimelineEventType.java | 3 +
.../timelineservice/NMTimelinePublisher.java | 14 +-
.../timelineservice/package-info.java | 29 ++++
.../resourcemanager/RMActiveServiceContext.java | 8 +-
.../server/resourcemanager/RMContextImpl.java | 4 +-
.../metrics/AbstractSystemMetricsPublisher.java | 20 ++-
.../metrics/NoOpSystemMetricPublisher.java | 2 +-
.../metrics/SystemMetricsPublisher.java | 3 +
.../metrics/TimelineServiceV1Publisher.java | 8 +-
.../metrics/TimelineServiceV2Publisher.java | 7 +-
.../resourcemanager/metrics/package-info.java | 28 +++
.../rmapp/RMAppCollectorUpdateEvent.java | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 5 +-
.../RMTimelineCollectorManager.java | 33 ++--
.../timelineservice/package-info.java | 28 +++
.../collector/AppLevelTimelineCollector.java | 3 +-
.../collector/NodeTimelineCollectorManager.java | 8 +-
.../PerNodeTimelineCollectorsAuxService.java | 2 +
.../collector/TimelineCollector.java | 2 +
.../collector/TimelineCollectorManager.java | 12 +-
.../collector/TimelineCollectorWebService.java | 70 +++++---
.../timelineservice/collector/package-info.java | 29 ++++
.../reader/TimelineReaderManager.java | 32 +++-
.../reader/TimelineReaderServer.java | 2 +-
.../reader/TimelineReaderWebServices.java | 28 +--
.../reader/TimelineReaderWebServicesUtils.java | 50 +++---
.../reader/TimelineUIDConverter.java | 10 +-
.../reader/filter/TimelineFilterUtils.java | 8 +-
.../timelineservice/reader/package-info.java | 6 +
.../storage/FileSystemTimelineReaderImpl.java | 70 ++++----
.../storage/FileSystemTimelineWriterImpl.java | 7 +-
.../storage/HBaseTimelineReaderImpl.java | 3 +
.../storage/HBaseTimelineWriterImpl.java | 29 ++--
.../storage/OfflineAggregationWriter.java | 13 +-
.../PhoenixOfflineAggregationWriterImpl.java | 27 +--
.../storage/TimelineAggregationTrack.java | 2 +-
.../timelineservice/storage/TimelineReader.java | 6 +-
.../storage/TimelineSchemaCreator.java | 4 +-
.../timelineservice/storage/TimelineWriter.java | 15 +-
.../storage/application/ApplicationColumn.java | 4 +-
.../application/ApplicationColumnPrefix.java | 8 +-
.../storage/application/ApplicationRowKey.java | 33 ++--
.../storage/application/ApplicationTable.java | 16 +-
.../storage/application/package-info.java | 4 +
.../storage/apptoflow/AppToFlowColumn.java | 6 +-
.../apptoflow/AppToFlowColumnFamily.java | 2 +-
.../storage/apptoflow/AppToFlowRowKey.java | 9 +-
.../storage/apptoflow/AppToFlowTable.java | 6 +-
.../storage/apptoflow/package-info.java | 5 +
.../storage/common/BaseTable.java | 21 ++-
.../common/BufferedMutatorDelegator.java | 2 +-
.../timelineservice/storage/common/Column.java | 16 +-
.../storage/common/ColumnFamily.java | 2 +-
.../storage/common/ColumnHelper.java | 24 +--
.../storage/common/ColumnPrefix.java | 22 ++-
.../storage/common/LongConverter.java | 5 +-
.../storage/common/NumericValueConverter.java | 7 +-
.../storage/common/OfflineAggregationInfo.java | 61 ++++---
.../timelineservice/storage/common/Range.java | 3 +
.../storage/common/Separator.java | 20 ++-
.../common/TimelineHBaseSchemaConstants.java | 33 ++--
.../storage/common/TimelineStorageUtils.java | 79 +++++----
.../storage/common/TimestampGenerator.java | 12 +-
.../storage/common/ValueConverter.java | 10 +-
.../storage/entity/EntityColumn.java | 2 +-
.../storage/entity/EntityColumnPrefix.java | 9 +-
.../storage/entity/EntityRowKey.java | 54 +++---
.../storage/entity/EntityTable.java | 16 +-
.../storage/entity/package-info.java | 4 +
.../flow/AggregationCompactionDimension.java | 6 +-
.../storage/flow/AggregationOperation.java | 10 +-
.../storage/flow/FlowActivityColumnFamily.java | 3 +-
.../storage/flow/FlowActivityColumnPrefix.java | 19 ++-
.../storage/flow/FlowActivityRowKey.java | 34 ++--
.../storage/flow/FlowActivityTable.java | 19 ++-
.../storage/flow/FlowRunColumnPrefix.java | 8 +-
.../storage/flow/FlowRunCoprocessor.java | 8 +-
.../storage/flow/FlowRunRowKey.java | 21 ++-
.../storage/flow/FlowRunTable.java | 8 +-
.../storage/flow/FlowScanner.java | 57 ++++---
.../storage/flow/package-info.java | 29 ++++
.../timelineservice/storage/package-info.java | 4 +
.../storage/reader/ApplicationEntityReader.java | 28 +--
.../reader/FlowActivityEntityReader.java | 2 +-
.../storage/reader/FlowRunEntityReader.java | 14 +-
.../storage/reader/GenericEntityReader.java | 58 +++++--
.../storage/reader/TimelineEntityReader.java | 80 ++++++++-
.../reader/TimelineEntityReaderFactory.java | 18 +-
.../storage/reader/package-info.java | 5 +
117 files changed, 1417 insertions(+), 818 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index a10872a..853c506 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -147,7 +147,8 @@ public class JobHistoryEventHandler extends AbstractService
private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB";
private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK";
- private static String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE = "MAPREDUCE_TASK_ATTEMPT";
+ private static final String MAPREDUCE_TASK_ATTEMPT_ENTITY_TYPE =
+ "MAPREDUCE_TASK_ATTEMPT";
public JobHistoryEventHandler(AppContext context, int startCount) {
super("JobHistoryEventHandler");
@@ -479,8 +480,9 @@ public class JobHistoryEventHandler extends AbstractService
try {
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
threadPool.shutdownNow();
- if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
+ if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
LOG.error("ThreadPool did not terminate");
+ }
}
} catch (InterruptedException ie) {
threadPool.shutdownNow();
@@ -1073,7 +1075,7 @@ public class JobHistoryEventHandler extends AbstractService
default:
break;
}
-
+
try {
TimelinePutResponse response = timelineClient.putEntities(tEntity);
List<TimelinePutResponse.TimelinePutError> errors = response.getErrors();
@@ -1116,12 +1118,12 @@ public class JobHistoryEventHandler extends AbstractService
return nodes;
}
- private void putEntityWithoutBlocking(final TimelineClient timelineClient,
+ private void putEntityWithoutBlocking(final TimelineClient client,
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity) {
Runnable publishWrapper = new Runnable() {
public void run() {
try {
- timelineClient.putEntities(entity);
+ client.putEntities(entity);
} catch (IOException|YarnException e) {
LOG.error("putEntityNonBlocking get failed: " + e);
throw new RuntimeException(e.toString());
@@ -1187,87 +1189,92 @@ public class JobHistoryEventHandler extends AbstractService
entity.addIsRelatedToEntity(relatedTaskEntity, taskId);
return entity;
}
-
- private void processEventForNewTimelineService(HistoryEvent event, JobId jobId,
- long timestamp) {
- org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity = null;
+
+ private void processEventForNewTimelineService(HistoryEvent event,
+ JobId jobId, long timestamp) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity =
+ null;
String taskId = null;
String taskAttemptId = null;
boolean setCreatedTime = false;
switch (event.getEventType()) {
- // Handle job events
- case JOB_SUBMITTED:
- setCreatedTime = true;
- break;
- case JOB_STATUS_CHANGED:
- case JOB_INFO_CHANGED:
- case JOB_INITED:
- case JOB_PRIORITY_CHANGED:
- case JOB_QUEUE_CHANGED:
- case JOB_FAILED:
- case JOB_KILLED:
- case JOB_ERROR:
- case JOB_FINISHED:
- case AM_STARTED:
- case NORMALIZED_RESOURCE:
- break;
- // Handle task events
- case TASK_STARTED:
- setCreatedTime = true;
- taskId = ((TaskStartedEvent)event).getTaskId().toString();
- break;
- case TASK_FAILED:
- taskId = ((TaskFailedEvent)event).getTaskId().toString();
- break;
- case TASK_UPDATED:
- taskId = ((TaskUpdatedEvent)event).getTaskId().toString();
- break;
- case TASK_FINISHED:
- taskId = ((TaskFinishedEvent)event).getTaskId().toString();
- break;
- case MAP_ATTEMPT_STARTED:
- case REDUCE_ATTEMPT_STARTED:
- setCreatedTime = true;
- taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
- taskAttemptId = ((TaskAttemptStartedEvent)event).
- getTaskAttemptId().toString();
- break;
- case CLEANUP_ATTEMPT_STARTED:
- case SETUP_ATTEMPT_STARTED:
- taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
- taskAttemptId = ((TaskAttemptStartedEvent)event).
- getTaskAttemptId().toString();
- break;
- case MAP_ATTEMPT_FAILED:
- case CLEANUP_ATTEMPT_FAILED:
- case REDUCE_ATTEMPT_FAILED:
- case SETUP_ATTEMPT_FAILED:
- case MAP_ATTEMPT_KILLED:
- case CLEANUP_ATTEMPT_KILLED:
- case REDUCE_ATTEMPT_KILLED:
- case SETUP_ATTEMPT_KILLED:
- taskId = ((TaskAttemptUnsuccessfulCompletionEvent)event).getTaskId().toString();
- taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event).
- getTaskAttemptId().toString();
- break;
- case MAP_ATTEMPT_FINISHED:
- taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString();
- taskAttemptId = ((MapAttemptFinishedEvent)event).getAttemptId().toString();
- break;
- case REDUCE_ATTEMPT_FINISHED:
- taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString();
- taskAttemptId = ((ReduceAttemptFinishedEvent)event).getAttemptId().toString();
- break;
- case SETUP_ATTEMPT_FINISHED:
- case CLEANUP_ATTEMPT_FINISHED:
- taskId = ((TaskAttemptFinishedEvent)event).getTaskId().toString();
- taskAttemptId = ((TaskAttemptFinishedEvent)event).getAttemptId().toString();
- break;
- default:
- LOG.warn("EventType: " + event.getEventType() + " cannot be recognized" +
- " and handled by timeline service.");
- return;
+ // Handle job events
+ case JOB_SUBMITTED:
+ setCreatedTime = true;
+ break;
+ case JOB_STATUS_CHANGED:
+ case JOB_INFO_CHANGED:
+ case JOB_INITED:
+ case JOB_PRIORITY_CHANGED:
+ case JOB_QUEUE_CHANGED:
+ case JOB_FAILED:
+ case JOB_KILLED:
+ case JOB_ERROR:
+ case JOB_FINISHED:
+ case AM_STARTED:
+ case NORMALIZED_RESOURCE:
+ break;
+ // Handle task events
+ case TASK_STARTED:
+ setCreatedTime = true;
+ taskId = ((TaskStartedEvent)event).getTaskId().toString();
+ break;
+ case TASK_FAILED:
+ taskId = ((TaskFailedEvent)event).getTaskId().toString();
+ break;
+ case TASK_UPDATED:
+ taskId = ((TaskUpdatedEvent)event).getTaskId().toString();
+ break;
+ case TASK_FINISHED:
+ taskId = ((TaskFinishedEvent)event).getTaskId().toString();
+ break;
+ case MAP_ATTEMPT_STARTED:
+ case REDUCE_ATTEMPT_STARTED:
+ setCreatedTime = true;
+ taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptStartedEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case CLEANUP_ATTEMPT_STARTED:
+ case SETUP_ATTEMPT_STARTED:
+ taskId = ((TaskAttemptStartedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptStartedEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case MAP_ATTEMPT_FAILED:
+ case CLEANUP_ATTEMPT_FAILED:
+ case REDUCE_ATTEMPT_FAILED:
+ case SETUP_ATTEMPT_FAILED:
+ case MAP_ATTEMPT_KILLED:
+ case CLEANUP_ATTEMPT_KILLED:
+ case REDUCE_ATTEMPT_KILLED:
+ case SETUP_ATTEMPT_KILLED:
+ taskId = ((TaskAttemptUnsuccessfulCompletionEvent)event).
+ getTaskId().toString();
+ taskAttemptId = ((TaskAttemptUnsuccessfulCompletionEvent)event).
+ getTaskAttemptId().toString();
+ break;
+ case MAP_ATTEMPT_FINISHED:
+ taskId = ((MapAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((MapAttemptFinishedEvent)event).
+ getAttemptId().toString();
+ break;
+ case REDUCE_ATTEMPT_FINISHED:
+ taskId = ((ReduceAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((ReduceAttemptFinishedEvent)event).
+ getAttemptId().toString();
+ break;
+ case SETUP_ATTEMPT_FINISHED:
+ case CLEANUP_ATTEMPT_FINISHED:
+ taskId = ((TaskAttemptFinishedEvent)event).getTaskId().toString();
+ taskAttemptId = ((TaskAttemptFinishedEvent)event).
+ getAttemptId().toString();
+ break;
+ default:
+ LOG.warn("EventType: " + event.getEventType() + " cannot be recognized" +
+ " and handled by timeline service.");
+ return;
}
if (taskId == null) {
// JobEntity
@@ -1286,7 +1293,6 @@ public class JobHistoryEventHandler extends AbstractService
taskId, setCreatedTime);
}
}
-
putEntityWithoutBlocking(timelineClient, tEntity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index e7d5006..6eb8a43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -405,7 +405,7 @@ public class TestEvents {
public void setDatum(Object datum) {
this.datum = datum;
}
-
+
@Override
public TimelineEvent toTimelineEvent() {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
index f4896ff..e60380b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -26,7 +26,12 @@ import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.ObjectNode;
-public class JobHistoryEventUtils {
+/**
+ * Class containing utility methods to be used by JobHistoryEventHandler.
+ */
+public final class JobHistoryEventUtils {
+ private JobHistoryEventUtils() {
+ }
public static JsonNode countersToJSON(Counters counters) {
ObjectMapper mapper = new ObjectMapper();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
index 80928dc..880014b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TimelineEntityConverter.java
@@ -125,7 +125,8 @@ class TimelineEntityConverter {
}
}
- private Set<TimelineEntity> createTaskAndTaskAttemptEntities(JobInfo jobInfo) {
+ private Set<TimelineEntity> createTaskAndTaskAttemptEntities(
+ JobInfo jobInfo) {
Set<TimelineEntity> entities = new HashSet<>();
Map<TaskID,TaskInfo> taskInfoMap = jobInfo.getAllTasks();
LOG.info("job " + jobInfo.getJobId()+ " has " + taskInfoMap.size() +
@@ -175,7 +176,8 @@ class TimelineEntityConverter {
return taskAttempts;
}
- private TimelineEntity createTaskAttemptEntity(TaskAttemptInfo taskAttemptInfo) {
+ private TimelineEntity createTaskAttemptEntity(
+ TaskAttemptInfo taskAttemptInfo) {
TimelineEntity taskAttempt = new TimelineEntity();
taskAttempt.setType(TASK_ATTEMPT);
taskAttempt.setId(taskAttemptInfo.getAttemptId().toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
index c290cd6..5d9dc0b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/JobHistoryFileParser.java
@@ -28,6 +28,9 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser;
import org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser.JobInfo;
+/**
+ * Used to parse job history and configuration files.
+ */
class JobHistoryFileParser {
private static final Log LOG = LogFactory.getLog(JobHistoryFileParser.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
index 734c741..053d84e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationAttemptEntity.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents an application attempt.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ApplicationAttemptEntity extends HierarchicalTimelineEntity {
@@ -29,8 +32,10 @@ public class ApplicationAttemptEntity extends HierarchicalTimelineEntity {
public ApplicationAttemptEntity(TimelineEntity entity) {
super(entity);
- if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ if (!entity.getType().equals(
+ TimelineEntityType.YARN_APPLICATION_ATTEMPT.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
index 183d8d8..6075ec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ApplicationEntity.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents an application.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ApplicationEntity extends HierarchicalTimelineEntity {
@@ -32,8 +35,10 @@ public class ApplicationEntity extends HierarchicalTimelineEntity {
public ApplicationEntity(TimelineEntity entity) {
super(entity);
- if (!entity.getType().equals(TimelineEntityType.YARN_APPLICATION.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ if (!entity.getType().equals(
+ TimelineEntityType.YARN_APPLICATION.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
index 94eefa8..1f96505 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ClusterEntity.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents a YARN cluster.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ClusterEntity extends HierarchicalTimelineEntity {
@@ -30,7 +33,8 @@ public class ClusterEntity extends HierarchicalTimelineEntity {
public ClusterEntity(TimelineEntity entity) {
super(entity);
if (!entity.getType().equals(TimelineEntityType.YARN_CLUSTER.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
index c7d6bce..f61920f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/ContainerEntity.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents a container belonging to an application attempt.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class ContainerEntity extends HierarchicalTimelineEntity {
@@ -29,8 +32,10 @@ public class ContainerEntity extends HierarchicalTimelineEntity {
public ContainerEntity(TimelineEntity entity) {
super(entity);
- if (!entity.getType().equals(TimelineEntityType.YARN_CONTAINER.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ if (!entity.getType().equals(
+ TimelineEntityType.YARN_CONTAINER.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
index 3c3ffb4..410a1bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/FlowRunEntity.java
@@ -22,6 +22,9 @@ import javax.xml.bind.annotation.XmlElement;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents a flow run.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class FlowRunEntity extends HierarchicalTimelineEntity {
@@ -44,8 +47,10 @@ public class FlowRunEntity extends HierarchicalTimelineEntity {
public FlowRunEntity(TimelineEntity entity) {
super(entity);
- if (!entity.getType().equals(TimelineEntityType.YARN_FLOW_RUN.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ if (!entity.getType().equals(
+ TimelineEntityType.YARN_FLOW_RUN.toString())) {
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
// set config to null
setConfigs(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
index 6235da0..4744e39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/HierarchicalTimelineEntity.java
@@ -17,17 +17,18 @@
*/
package org.apache.hadoop.yarn.api.records.timelineservice;
-import com.google.common.base.Joiner;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
-import java.util.List;
import java.util.Set;
+/**
+ * This class extends timeline entity and defines parent-child relationships
+ * with other entities.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public abstract class HierarchicalTimelineEntity extends TimelineEntity {
@@ -66,6 +67,7 @@ public abstract class HierarchicalTimelineEntity extends TimelineEntity {
setParent(new Identifier(type, id));
}
+ @SuppressWarnings("unchecked")
public Set<Identifier> getChildren() {
Object identifiers = getInfo().get(CHILDREN_INFO_KEY);
if (identifiers == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
index d1a1f19..b654450 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/QueueEntity.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents a queue.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class QueueEntity extends HierarchicalTimelineEntity {
@@ -30,7 +33,8 @@ public class QueueEntity extends HierarchicalTimelineEntity {
public QueueEntity(TimelineEntity entity) {
super(entity);
if (!entity.getType().equals(TimelineEntityType.YARN_QUEUE.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
index 39504cc..f08a0ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntities.java
@@ -27,6 +27,9 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashSet;
import java.util.Set;
+/**
+ * This class hosts a set of timeline entities.
+ */
@XmlRootElement(name = "entities")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@@ -44,12 +47,12 @@ public class TimelineEntities {
return entities;
}
- public void setEntities(Set<TimelineEntity> entities) {
- this.entities = entities;
+ public void setEntities(Set<TimelineEntity> timelineEntities) {
+ this.entities = timelineEntities;
}
- public void addEntities(Set<TimelineEntity> entities) {
- this.entities.addAll(entities);
+ public void addEntities(Set<TimelineEntity> timelineEntities) {
+ this.entities.addAll(timelineEntities);
}
public void addEntity(TimelineEntity entity) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index a661f7a..acc132e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -54,6 +54,9 @@ import java.util.TreeSet;
public class TimelineEntity implements Comparable<TimelineEntity> {
protected final static String SYSTEM_INFO_KEY_PREFIX = "SYSTEM_INFO_";
+ /**
+ * Identifier of timeline entity(entity id + entity type).
+ */
@XmlRootElement(name = "identifier")
@XmlAccessorType(XmlAccessType.NONE)
public static class Identifier {
@@ -74,8 +77,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
return type;
}
- public void setType(String type) {
- this.type = type;
+ public void setType(String entityType) {
+ this.type = entityType;
}
@XmlElement(name = "id")
@@ -83,8 +86,8 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
return id;
}
- public void setId(String id) {
- this.id = id;
+ public void setId(String entityId) {
+ this.id = entityId;
}
@Override
@@ -106,8 +109,9 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
+ }
if (!(obj instanceof Identifier)) {
return false;
}
@@ -208,11 +212,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setIdentifier(Identifier identifier) {
+ public void setIdentifier(Identifier entityIdentifier) {
if (real == null) {
- this.identifier = identifier;
+ this.identifier = entityIdentifier;
} else {
- real.setIdentifier(identifier);
+ real.setIdentifier(entityIdentifier);
}
}
@@ -235,19 +239,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setInfo(Map<String, Object> info) {
+ public void setInfo(Map<String, Object> entityInfos) {
if (real == null) {
- this.info = TimelineServiceHelper.mapCastToHashMap(info);
+ this.info = TimelineServiceHelper.mapCastToHashMap(entityInfos);
} else {
- real.setInfo(info);
+ real.setInfo(entityInfos);
}
}
- public void addInfo(Map<String, Object> info) {
+ public void addInfo(Map<String, Object> entityInfos) {
if (real == null) {
- this.info.putAll(info);
+ this.info.putAll(entityInfos);
} else {
- real.addInfo(info);
+ real.addInfo(entityInfos);
}
}
@@ -278,19 +282,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setConfigs(Map<String, String> configs) {
+ public void setConfigs(Map<String, String> entityConfigs) {
if (real == null) {
- this.configs = TimelineServiceHelper.mapCastToHashMap(configs);
+ this.configs = TimelineServiceHelper.mapCastToHashMap(entityConfigs);
} else {
- real.setConfigs(configs);
+ real.setConfigs(entityConfigs);
}
}
- public void addConfigs(Map<String, String> configs) {
+ public void addConfigs(Map<String, String> entityConfigs) {
if (real == null) {
- this.configs.putAll(configs);
+ this.configs.putAll(entityConfigs);
} else {
- real.addConfigs(configs);
+ real.addConfigs(entityConfigs);
}
}
@@ -311,19 +315,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setMetrics(Set<TimelineMetric> metrics) {
+ public void setMetrics(Set<TimelineMetric> entityMetrics) {
if (real == null) {
- this.metrics = metrics;
+ this.metrics = entityMetrics;
} else {
- real.setMetrics(metrics);
+ real.setMetrics(entityMetrics);
}
}
- public void addMetrics(Set<TimelineMetric> metrics) {
+ public void addMetrics(Set<TimelineMetric> entityMetrics) {
if (real == null) {
- this.metrics.addAll(metrics);
+ this.metrics.addAll(entityMetrics);
} else {
- real.addMetrics(metrics);
+ real.addMetrics(entityMetrics);
}
}
@@ -344,19 +348,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void setEvents(NavigableSet<TimelineEvent> events) {
+ public void setEvents(NavigableSet<TimelineEvent> entityEvents) {
if (real == null) {
- this.events = events;
+ this.events = entityEvents;
} else {
- real.setEvents(events);
+ real.setEvents(entityEvents);
}
}
- public void addEvents(Set<TimelineEvent> events) {
+ public void addEvents(Set<TimelineEvent> entityEvents) {
if (real == null) {
- this.events.addAll(events);
+ this.events.addAll(entityEvents);
} else {
- real.addEvents(events);
+ real.addEvents(entityEvents);
}
}
@@ -389,20 +393,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
@JsonSetter("isrelatedto")
public void setIsRelatedToEntities(
- Map<String, Set<String>> isRelatedToEntities) {
+ Map<String, Set<String>> isRelatedTo) {
if (real == null) {
this.isRelatedToEntities =
- TimelineServiceHelper.mapCastToHashMap(isRelatedToEntities);
+ TimelineServiceHelper.mapCastToHashMap(isRelatedTo);
} else {
- real.setIsRelatedToEntities(isRelatedToEntities);
+ real.setIsRelatedToEntities(isRelatedTo);
}
}
public void addIsRelatedToEntities(
- Map<String, Set<String>> isRelatedToEntities) {
+ Map<String, Set<String>> isRelatedTo) {
if (real == null) {
- for (Map.Entry<String, Set<String>> entry : isRelatedToEntities
- .entrySet()) {
+ for (Map.Entry<String, Set<String>> entry : isRelatedTo.entrySet()) {
Set<String> ids = this.isRelatedToEntities.get(entry.getKey());
if (ids == null) {
ids = new HashSet<>();
@@ -411,7 +414,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
ids.addAll(entry.getValue());
}
} else {
- real.addIsRelatedToEntities(isRelatedToEntities);
+ real.addIsRelatedToEntities(isRelatedTo);
}
}
@@ -447,10 +450,9 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
- public void addRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
+ public void addRelatesToEntities(Map<String, Set<String>> relatesTo) {
if (real == null) {
- for (Map.Entry<String, Set<String>> entry : relatesToEntities
- .entrySet()) {
+ for (Map.Entry<String, Set<String>> entry : relatesTo.entrySet()) {
Set<String> ids = this.relatesToEntities.get(entry.getKey());
if (ids == null) {
ids = new HashSet<>();
@@ -459,7 +461,7 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
ids.addAll(entry.getValue());
}
} else {
- real.addRelatesToEntities(relatesToEntities);
+ real.addRelatesToEntities(relatesTo);
}
}
@@ -477,12 +479,12 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
@JsonSetter("relatesto")
- public void setRelatesToEntities(Map<String, Set<String>> relatesToEntities) {
+ public void setRelatesToEntities(Map<String, Set<String>> relatesTo) {
if (real == null) {
this.relatesToEntities =
- TimelineServiceHelper.mapCastToHashMap(relatesToEntities);
+ TimelineServiceHelper.mapCastToHashMap(relatesTo);
} else {
- real.setRelatesToEntities(relatesToEntities);
+ real.setRelatesToEntities(relatesTo);
}
}
@@ -496,11 +498,11 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
@JsonSetter("createdtime")
- public void setCreatedTime(long createdTime) {
+ public void setCreatedTime(long createdTs) {
if (real == null) {
- this.createdTime = createdTime;
+ this.createdTime = createdTs;
} else {
- real.setCreatedTime(createdTime);
+ real.setCreatedTime(createdTs);
}
}
@@ -530,10 +532,12 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
@Override
public boolean equals(Object obj) {
- if (this == obj)
+ if (this == obj) {
return true;
- if (!(obj instanceof TimelineEntity))
+ }
+ if (!(obj instanceof TimelineEntity)) {
return false;
+ }
TimelineEntity other = (TimelineEntity) obj;
return getIdentifier().equals(other.getIdentifier());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
index ba32e20..8fcc2ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntityType.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * Defines type of entity.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public enum TimelineEntityType {
@@ -34,51 +37,63 @@ public enum TimelineEntityType {
/**
* Whether the input type can be a parent of this entity.
+ *
+ * @param type entity type.
+ * @return true, if this entity type is parent of passed entity type, false
+ * otherwise.
*/
public boolean isParent(TimelineEntityType type) {
switch (this) {
- case YARN_CLUSTER:
- return false;
- case YARN_FLOW_RUN:
- return YARN_FLOW_RUN == type || YARN_CLUSTER == type;
- case YARN_APPLICATION:
- return YARN_FLOW_RUN == type || YARN_CLUSTER == type;
- case YARN_APPLICATION_ATTEMPT:
- return YARN_APPLICATION == type;
- case YARN_CONTAINER:
- return YARN_APPLICATION_ATTEMPT == type;
- case YARN_QUEUE:
- return YARN_QUEUE == type;
- default:
- return false;
+ case YARN_CLUSTER:
+ return false;
+ case YARN_FLOW_RUN:
+ return YARN_FLOW_RUN == type || YARN_CLUSTER == type;
+ case YARN_APPLICATION:
+ return YARN_FLOW_RUN == type || YARN_CLUSTER == type;
+ case YARN_APPLICATION_ATTEMPT:
+ return YARN_APPLICATION == type;
+ case YARN_CONTAINER:
+ return YARN_APPLICATION_ATTEMPT == type;
+ case YARN_QUEUE:
+ return YARN_QUEUE == type;
+ default:
+ return false;
}
}
/**
* Whether the input type can be a child of this entity.
+ *
+ * @param type entity type.
+ * @return true, if this entity type is child of passed entity type, false
+ * otherwise.
*/
public boolean isChild(TimelineEntityType type) {
switch (this) {
- case YARN_CLUSTER:
- return YARN_FLOW_RUN == type || YARN_APPLICATION == type;
- case YARN_FLOW_RUN:
- return YARN_FLOW_RUN == type || YARN_APPLICATION == type;
- case YARN_APPLICATION:
- return YARN_APPLICATION_ATTEMPT == type;
- case YARN_APPLICATION_ATTEMPT:
- return YARN_CONTAINER == type;
- case YARN_CONTAINER:
- return false;
- case YARN_QUEUE:
- return YARN_QUEUE == type;
- default:
- return false;
+ case YARN_CLUSTER:
+ return YARN_FLOW_RUN == type || YARN_APPLICATION == type;
+ case YARN_FLOW_RUN:
+ return YARN_FLOW_RUN == type || YARN_APPLICATION == type;
+ case YARN_APPLICATION:
+ return YARN_APPLICATION_ATTEMPT == type;
+ case YARN_APPLICATION_ATTEMPT:
+ return YARN_CONTAINER == type;
+ case YARN_CONTAINER:
+ return false;
+ case YARN_QUEUE:
+ return YARN_QUEUE == type;
+ default:
+ return false;
}
}
/**
* Whether the type of this entity matches the type indicated by the input
* argument.
+ *
+ * @param typeString entity type represented as a string.
+ * @return true, if string representation of this entity type matches the
+ * entity type passed.
*/
public boolean matches(String typeString) {
return toString().equals(typeString);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
index a563658..87fc291 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEvent.java
@@ -28,6 +28,11 @@ import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashMap;
import java.util.Map;
+/**
+ * This class contains the information of an event that belongs to an entity.
+ * Users are free to define what the event means, such as starting an
+ * application, container being allocated, etc.
+ */
@XmlRootElement(name = "event")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@@ -48,8 +53,8 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
return id;
}
- public void setId(String id) {
- this.id = id;
+ public void setId(String eventId) {
+ this.id = eventId;
}
// required by JAXB
@@ -63,12 +68,12 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
return info;
}
- public void setInfo(Map<String, Object> info) {
- this.info = TimelineServiceHelper.mapCastToHashMap(info);
+ public void setInfo(Map<String, Object> infos) {
+ this.info = TimelineServiceHelper.mapCastToHashMap(infos);
}
- public void addInfo(Map<String, Object> info) {
- this.info.putAll(info);
+ public void addInfo(Map<String, Object> infos) {
+ this.info.putAll(infos);
}
public void addInfo(String key, Object value) {
@@ -80,8 +85,8 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
return timestamp;
}
- public void setTimestamp(long timestamp) {
- this.timestamp = timestamp;
+ public void setTimestamp(long ts) {
+ this.timestamp = ts;
}
public boolean isValid() {
@@ -97,15 +102,18 @@ public class TimelineEvent implements Comparable<TimelineEvent> {
@Override
public boolean equals(Object o) {
- if (this == o)
+ if (this == o) {
return true;
- if (!(o instanceof TimelineEvent))
+ }
+ if (!(o instanceof TimelineEvent)) {
return false;
+ }
TimelineEvent event = (TimelineEvent) o;
- if (timestamp != event.timestamp)
+ if (timestamp != event.timestamp) {
return false;
+ }
if (!id.equals(event.id)) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index e3870da..2f60515 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -28,12 +28,19 @@ import java.util.Comparator;
import java.util.Map;
import java.util.TreeMap;
+/**
+ * This class contains the information of a metric that is related to some
+ * entity. Metric can either be a time series or single value.
+ */
@XmlRootElement(name = "metric")
@XmlAccessorType(XmlAccessType.NONE)
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class TimelineMetric {
+ /**
+ * Type of metric.
+ */
public static enum Type {
SINGLE_VALUE,
TIME_SERIES
@@ -63,8 +70,8 @@ public class TimelineMetric {
return type;
}
- public void setType(Type type) {
- this.type = type;
+ public void setType(Type metricType) {
+ this.type = metricType;
}
@XmlElement(name = "id")
@@ -72,8 +79,8 @@ public class TimelineMetric {
return id;
}
- public void setId(String id) {
- this.id = id;
+ public void setId(String metricId) {
+ this.id = metricId;
}
// required by JAXB
@@ -87,24 +94,24 @@ public class TimelineMetric {
return values;
}
- public void setValues(Map<Long, Number> values) {
+ public void setValues(Map<Long, Number> vals) {
if (type == Type.SINGLE_VALUE) {
- overwrite(values);
+ overwrite(vals);
} else {
if (values != null) {
this.values = new TreeMap<Long, Number>(reverseComparator);
- this.values.putAll(values);
+ this.values.putAll(vals);
} else {
this.values = null;
}
}
}
- public void addValues(Map<Long, Number> values) {
+ public void addValues(Map<Long, Number> vals) {
if (type == Type.SINGLE_VALUE) {
- overwrite(values);
+ overwrite(vals);
} else {
- this.values.putAll(values);
+ this.values.putAll(vals);
}
}
@@ -115,14 +122,14 @@ public class TimelineMetric {
values.put(timestamp, value);
}
- private void overwrite(Map<Long, Number> values) {
- if (values.size() > 1) {
+ private void overwrite(Map<Long, Number> vals) {
+ if (vals.size() > 1) {
throw new IllegalArgumentException(
"Values cannot contain more than one point in " +
Type.SINGLE_VALUE + " mode");
}
this.values.clear();
- this.values.putAll(values);
+ this.values.putAll(vals);
}
public boolean isValid() {
@@ -139,10 +146,12 @@ public class TimelineMetric {
// Only check if type and id are equal
@Override
public boolean equals(Object o) {
- if (this == o)
+ if (this == o) {
return true;
- if (!(o instanceof TimelineMetric))
+ }
+ if (!(o instanceof TimelineMetric)) {
return false;
+ }
TimelineMetric m = (TimelineMetric) o;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
index 4739d8f..eda1ee2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineWriteResponse.java
@@ -29,9 +29,9 @@ import java.util.List;
/**
* A class that holds a list of put errors. This is the response returned when a
- * list of {@link TimelineEntity} objects is added to the timeline. If there are errors
- * in storing individual entity objects, they will be indicated in the list of
- * errors.
+ * list of {@link TimelineEntity} objects is added to the timeline. If there are
+ * errors in storing individual entity objects, they will be indicated in the
+ * list of errors.
*/
@XmlRootElement(name = "response")
@XmlAccessorType(XmlAccessType.NONE)
@@ -46,7 +46,7 @@ public class TimelineWriteResponse {
}
/**
- * Get a list of {@link TimelineWriteError} instances
+ * Get a list of {@link TimelineWriteError} instances.
*
* @return a list of {@link TimelineWriteError} instances
*/
@@ -56,7 +56,7 @@ public class TimelineWriteResponse {
}
/**
- * Add a single {@link TimelineWriteError} instance into the existing list
+ * Add a single {@link TimelineWriteError} instance into the existing list.
*
* @param error
* a single {@link TimelineWriteError} instance
@@ -66,24 +66,24 @@ public class TimelineWriteResponse {
}
/**
- * Add a list of {@link TimelineWriteError} instances into the existing list
+ * Add a list of {@link TimelineWriteError} instances into the existing list.
*
- * @param errors
+ * @param writeErrors
* a list of {@link TimelineWriteError} instances
*/
- public void addErrors(List<TimelineWriteError> errors) {
- this.errors.addAll(errors);
+ public void addErrors(List<TimelineWriteError> writeErrors) {
+ this.errors.addAll(writeErrors);
}
/**
- * Set the list to the given list of {@link TimelineWriteError} instances
+ * Set the list to the given list of {@link TimelineWriteError} instances.
*
- * @param errors
+ * @param writeErrors
* a list of {@link TimelineWriteError} instances
*/
- public void setErrors(List<TimelineWriteError> errors) {
+ public void setErrors(List<TimelineWriteError> writeErrors) {
this.errors.clear();
- this.errors.addAll(errors);
+ this.errors.addAll(writeErrors);
}
/**
@@ -106,7 +106,7 @@ public class TimelineWriteResponse {
private int errorCode;
/**
- * Get the entity Id
+ * Get the entity Id.
*
* @return the entity Id
*/
@@ -116,17 +116,16 @@ public class TimelineWriteResponse {
}
/**
- * Set the entity Id
+ * Set the entity Id.
*
- * @param entityId
- * the entity Id
+ * @param id the entity Id.
*/
- public void setEntityId(String entityId) {
- this.entityId = entityId;
+ public void setEntityId(String id) {
+ this.entityId = id;
}
/**
- * Get the entity type
+ * Get the entity type.
*
* @return the entity type
*/
@@ -136,17 +135,16 @@ public class TimelineWriteResponse {
}
/**
- * Set the entity type
+ * Set the entity type.
*
- * @param entityType
- * the entity type
+ * @param type the entity type.
*/
- public void setEntityType(String entityType) {
- this.entityType = entityType;
+ public void setEntityType(String type) {
+ this.entityType = type;
}
/**
- * Get the error code
+ * Get the error code.
*
* @return an error code
*/
@@ -156,13 +154,12 @@ public class TimelineWriteResponse {
}
/**
- * Set the error code to the given error code
+ * Set the error code to the given error code.
*
- * @param errorCode
- * an error code
+ * @param code an error code.
*/
- public void setErrorCode(int errorCode) {
- this.errorCode = errorCode;
+ public void setErrorCode(int code) {
+ this.errorCode = code;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
index a229fd8..ced57c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/UserEntity.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * This entity represents a user.
+ */
@InterfaceAudience.Public
@InterfaceStability.Unstable
public class UserEntity extends TimelineEntity {
@@ -30,7 +33,8 @@ public class UserEntity extends TimelineEntity {
public UserEntity(TimelineEntity entity) {
super(entity);
if (!entity.getType().equals(TimelineEntityType.YARN_USER.toString())) {
- throw new IllegalArgumentException("Incompatible entity type: " + getId());
+ throw new IllegalArgumentException("Incompatible entity type: "
+ + getId());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java
index 89a9e9b..c43bd62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/package-info.java
@@ -15,7 +15,13 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-@InterfaceAudience.Public package org.apache.hadoop.yarn.api.records.timelineservice;
+
+/**
+ * Package org.apache.hadoop.yarn.api.records.timelineservice contains classes
+ * which define the data model for ATSv2.
+ */
+@InterfaceAudience.Public
+package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8ee417b..2490dda 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -465,7 +465,8 @@ public class YarnConfiguration extends Configuration {
*/
public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = RM_PREFIX
+ "system-metrics-publisher.enabled";
- public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED = false;
+ public static final boolean DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_ENABLED =
+ false;
/**
* The setting that controls whether yarn system metrics is published on the
@@ -487,8 +488,8 @@ public class YarnConfiguration extends Configuration {
public static final String RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
RM_PREFIX + "system-metrics-publisher.dispatcher.pool-size";
- public static final int DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE =
- 10;
+ public static final int
+ DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10;
//RM delegation token related keys
public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
@@ -1753,7 +1754,7 @@ public class YarnConfiguration extends Configuration {
public static final String TIMELINE_SERVICE_UI_WEB_PATH_PREFIX =
TIMELINE_SERVICE_PREFIX + "ui-web-path.";
- /** Timeline client settings */
+ /** Timeline client settings. */
public static final String TIMELINE_SERVICE_CLIENT_PREFIX =
TIMELINE_SERVICE_PREFIX + "client.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index e0f2267..58307d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -55,7 +55,7 @@ public abstract class TimelineClient extends AbstractService implements
* construct and initialize a timeline client if the following operations are
* supposed to be conducted by that user.
*/
- protected ApplicationId contextAppId;
+ private ApplicationId contextAppId;
/**
* Creates an instance of the timeline v.1.x client.
@@ -78,7 +78,7 @@ public abstract class TimelineClient extends AbstractService implements
@Private
protected TimelineClient(String name, ApplicationId appId) {
super(name);
- contextAppId = appId;
+ setContextAppId(appId);
}
/**
@@ -242,11 +242,18 @@ public abstract class TimelineClient extends AbstractService implements
/**
* <p>
- * Update the timeline service address where the request will be sent to
+ * Update the timeline service address where the request will be sent to.
* </p>
* @param address
* the timeline service address
*/
public abstract void setTimelineServiceAddress(String address);
+ protected ApplicationId getContextAppId() {
+ return contextAppId;
+ }
+
+ protected void setContextAppId(ApplicationId appId) {
+ this.contextAppId = appId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 4db3b71..06b3ac4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -400,8 +400,8 @@ public class TimelineClientImpl extends TimelineClient {
entitiesContainer.addEntity(entity);
}
MultivaluedMap<String, String> params = new MultivaluedMapImpl();
- if (contextAppId != null) {
- params.add("appid", contextAppId.toString());
+ if (getContextAppId() != null) {
+ params.add("appid", getContextAppId().toString());
}
if (async) {
params.add("async", Boolean.TRUE.toString());
@@ -439,8 +439,7 @@ public class TimelineClientImpl extends TimelineClient {
URI uri = constructResURI(getConfig(), timelineServiceAddress, true);
putObjects(uri, path, params, obj);
needRetry = false;
- }
- catch (Exception e) {
+ } catch (Exception e) {
// TODO only handle exception for timelineServiceAddress being updated.
// skip retry for other exceptions.
checkRetryWithSleep(retries, e);
@@ -463,10 +462,9 @@ public class TimelineClientImpl extends TimelineClient {
Thread.currentThread().interrupt();
}
} else {
- LOG.error(
- "TimelineClient has reached to max retry times :" +
- this.maxServiceRetries + " for service address: " +
- timelineServiceAddress);
+ LOG.error("TimelineClient has reached to max retry times :" +
+ this.maxServiceRetries + " for service address: " +
+ timelineServiceAddress);
if (e instanceof YarnException) {
throw (YarnException)e;
} else if (e instanceof IOException) {
@@ -632,7 +630,7 @@ public class TimelineClientImpl extends TimelineClient {
}
/**
- * Poll TimelineServiceAddress for maximum of retries times if it is null
+ * Poll TimelineServiceAddress for maximum of retries times if it is null.
* @param retries
* @return the left retry times
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index 150026e..6faeffa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -45,8 +45,10 @@ import org.codehaus.jackson.map.ObjectMapper;
public class TimelineUtils {
public static final String FLOW_NAME_TAG_PREFIX = "TIMELINE_FLOW_NAME_TAG";
- public static final String FLOW_VERSION_TAG_PREFIX = "TIMELINE_FLOW_VERSION_TAG";
- public static final String FLOW_RUN_ID_TAG_PREFIX = "TIMELINE_FLOW_RUN_ID_TAG";
+ public static final String FLOW_VERSION_TAG_PREFIX =
+ "TIMELINE_FLOW_VERSION_TAG";
+ public static final String FLOW_RUN_ID_TAG_PREFIX =
+ "TIMELINE_FLOW_RUN_ID_TAG";
private static ObjectMapper mapper;
@@ -160,38 +162,39 @@ public class TimelineUtils {
return SecurityUtil.buildTokenService(timelineServiceAddr);
}
- public static String generateDefaultFlowNameBasedOnAppId(ApplicationId appId) {
+ public static String generateDefaultFlowNameBasedOnAppId(
+ ApplicationId appId) {
return "flow_" + appId.getClusterTimestamp() + "_" + appId.getId();
}
/**
- * Generate flow name tag
+ * Generate flow name tag.
*
* @param flowName flow name that identifies a distinct flow application which
* can be run repeatedly over time
- * @return
+ * @return flow name tag.
*/
public static String generateFlowNameTag(String flowName) {
return FLOW_NAME_TAG_PREFIX + ":" + flowName;
}
/**
- * Generate flow version tag
+ * Generate flow version tag.
*
* @param flowVersion flow version that keeps track of the changes made to the
* flow
- * @return
+ * @return flow version tag.
*/
public static String generateFlowVersionTag(String flowVersion) {
return FLOW_VERSION_TAG_PREFIX + ":" + flowVersion;
}
/**
- * Generate flow run ID tag
+ * Generate flow run ID tag.
*
* @param flowRunId flow run ID that identifies one instance (or specific
* execution) of that flow
- * @return
+ * @return flow run id tag.
*/
public static String generateFlowRunIdTag(long flowRunId) {
return FLOW_RUN_ID_TAG_PREFIX + ":" + flowRunId;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index b41a81a..415130b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -476,7 +476,7 @@ public class NodeManager extends CompositeService
protected final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
- protected Map<ApplicationId, String> registeredCollectors;
+ private Map<ApplicationId, String> registeredCollectors;
protected final ConcurrentMap<ContainerId,
org.apache.hadoop.yarn.api.records.Container> increasedContainers =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index 3ba81ce..548c861 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -52,12 +52,11 @@ public class NMCollectorService extends CompositeService implements
private static final Log LOG = LogFactory.getLog(NMCollectorService.class);
- final Context context;
+ private final Context context;
private Server server;
public NMCollectorService(Context context) {
-
super(NMCollectorService.class.getName());
this.context = context;
}
@@ -123,7 +122,8 @@ public class NMCollectorService extends CompositeService implements
client.setTimelineServiceAddress(collectorAddr);
}
}
- ((NodeManager.NMContext)context).addRegisteredCollectors(newCollectorsMap);
+ ((NodeManager.NMContext)context).addRegisteredCollectors(
+ newCollectorsMap);
}
return ReportNewCollectorInfoResponse.newInstance();
@@ -139,6 +139,7 @@ public class NMCollectorService extends CompositeService implements
" doesn't exist on NM.");
}
return GetTimelineCollectorContextResponse.newInstance(
- app.getUser(), app.getFlowName(), app.getFlowVersion(), app.getFlowRunId());
+ app.getUser(), app.getFlowName(), app.getFlowVersion(),
+ app.getFlowRunId());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java
new file mode 100644
index 0000000..7bf597b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.nodemanager.collectormanager contains
+ * classes for handling timeline collector information.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.nodemanager.collectormanager;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
index af8d94c..f275b37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEvent.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
import org.apache.hadoop.yarn.event.AbstractEvent;
+/**
+ * Event posted to NMTimelinePublisher which in turn publishes it to
+ * timelineservice v2.
+ */
public class NMTimelineEvent extends AbstractEvent<NMTimelineEventType> {
public NMTimelineEvent(NMTimelineEventType type) {
super(type);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
index c1129af..b4ae45a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelineEventType.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+/**
+ * Type of {@link NMTimelineEvent}.
+ */
public enum NMTimelineEventType {
// Publish the NM Timeline entity
TIMELINE_ENTITY_PUBLISH,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index affaaae..684feaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -127,7 +127,8 @@ public class NMTimelinePublisher extends CompositeService {
memoryMetric.addValue(currentTimeMillis, pmemUsage);
entity.addMetric(memoryMetric);
}
- if (cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE) {
+ if (cpuUsageTotalCoresPercentage !=
+ ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
cpuMetric.addValue(currentTimeMillis, cpuUsageTotalCoresPercentage);
@@ -189,7 +190,8 @@ public class NMTimelinePublisher extends CompositeService {
putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
}
- private static ContainerEntity createContainerEntity(ContainerId containerId) {
+ private static ContainerEntity createContainerEntity(
+ ContainerId containerId) {
ContainerEntity entity = new ContainerEntity();
entity.setId(containerId.toString());
Identifier parentIdentifier = new Identifier();
@@ -214,6 +216,7 @@ public class NMTimelinePublisher extends CompositeService {
}
}
+ @SuppressWarnings("unchecked")
public void publishApplicationEvent(ApplicationEvent event) {
// publish only when the desired event is received
switch (event.getType()) {
@@ -226,14 +229,14 @@ public class NMTimelinePublisher extends CompositeService {
default:
if (LOG.isDebugEnabled()) {
- LOG.debug(event.getType()
- + " is not a desired ApplicationEvent which needs to be published by"
- + " NMTimelinePublisher");
+ LOG.debug(event.getType() + " is not a desired ApplicationEvent which"
+ + " needs to be published by NMTimelinePublisher");
}
break;
}
}
+ @SuppressWarnings("unchecked")
public void publishContainerEvent(ContainerEvent event) {
// publish only when the desired event is received
switch (event.getType()) {
@@ -251,6 +254,7 @@ public class NMTimelinePublisher extends CompositeService {
}
}
+ @SuppressWarnings("unchecked")
public void publishLocalizationEvent(LocalizationEvent event) {
// publish only when the desired event is received
switch (event.getType()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java
new file mode 100644
index 0000000..66233fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.nodemanager.timelineservice contains
+ * classes related to publishing container events and other NM lifecycle events
+ * to ATSv2.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-4062. Add the flush and
compaction functionality via coprocessors and scanners for flow run table
(Vrushali C via sjlee)
Posted by gt...@apache.org.
YARN-4062. Add the flush and compaction functionality via coprocessors and scanners for flow run table (Vrushali C via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3157d32a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3157d32a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3157d32a
Branch: refs/heads/YARN-2928
Commit: 3157d32ae8ed526430beb85a4920243d38bd2b41
Parents: cd44408
Author: Sangjin Lee <sj...@apache.org>
Authored: Thu Mar 17 18:22:04 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:14 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 16 +
.../src/main/resources/yarn-default.xml | 10 +
.../storage/HBaseTimelineWriterImpl.java | 5 +-
.../storage/common/TimelineStorageUtils.java | 55 ++
.../storage/common/TimestampGenerator.java | 13 +-
.../storage/flow/AggregationOperation.java | 17 +-
.../storage/flow/FlowRunColumn.java | 4 +-
.../storage/flow/FlowRunColumnPrefix.java | 2 +-
.../storage/flow/FlowRunCoprocessor.java | 70 +-
.../storage/flow/FlowRunRowKey.java | 16 +
.../storage/flow/FlowScanner.java | 269 ++++++--
.../storage/flow/FlowScannerOperation.java | 46 ++
.../storage/flow/TestFlowDataGenerator.java | 178 +++++-
.../storage/flow/TestHBaseStorageFlowRun.java | 112 +++-
.../flow/TestHBaseStorageFlowRunCompaction.java | 635 +++++++++++++++++++
15 files changed, 1362 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 5115758..b79b3a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1900,6 +1900,22 @@ public class YarnConfiguration extends Configuration {
public static final int
DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
+ /**
+ * The name for setting that controls how long the final value of
+ * a metric of a completed app is retained before merging
+ * into the flow sum.
+ */
+ public static final String APP_FINAL_VALUE_RETENTION_THRESHOLD =
+ TIMELINE_SERVICE_PREFIX
+ + "coprocessor.app-final-value-retention-milliseconds";
+
+ /**
+ * The setting that controls how long the final value of a metric
+ * of a completed app is retained before merging into the flow sum.
+ */
+ public static final long DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD = 3 * 24
+ * 60 * 60 * 1000L;
+
public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS =
TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index ca913b8..13d4b4f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2128,6 +2128,7 @@
<value>604800</value>
</property>
+ <!-- Timeline Service v2 Configuration -->
<property>
<name>yarn.timeline-service.entity-group-fs-store.leveldb-cache-read-cache-size</name>
<description>
@@ -2196,6 +2197,15 @@
<name>yarn.timeline-service.timeline-client.number-of-async-entities-to-merge</name>
<value>10</value>
</property>
+
+ <property>
+ <description> The setting that controls how long the final value
+ of a metric of a completed app is retained before merging into
+ the flow sum.</description>
+ <name>yarn.timeline-service.coprocessor.app-final-value-retention-milliseconds</name>
+ <value>259200000</value>
+ </property>
+
<!-- Shared Cache Configuration -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 1afe878..b75007d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -261,7 +261,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
byte[] rowKey = FlowRunRowKey.getRowKey(clusterId, userId, flowName,
flowRunId);
storeFlowMetrics(rowKey, metrics,
- AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId));
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId),
+ AggregationOperation.SUM.getAttribute());
}
}
@@ -500,4 +501,4 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
super.serviceStop();
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index 605dbe7..b5fc214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -24,9 +24,13 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
+import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -475,4 +479,55 @@ public final class TimelineStorageUtils {
return (obj instanceof Short) || (obj instanceof Integer) ||
(obj instanceof Long);
}
+
+ /**
+ * creates a new cell based on the input cell but with the new value.
+ *
+ * @param origCell Original cell
+ * @param newValue new cell value
+ * @return cell
+ * @throws IOException while creating new cell.
+ */
+ public static Cell createNewCell(Cell origCell, byte[] newValue)
+ throws IOException {
+ return CellUtil.createCell(CellUtil.cloneRow(origCell),
+ CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell),
+ origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue);
+ }
+
+ /**
+ * creates a cell with the given inputs.
+ *
+ * @param row row of the cell to be created
+ * @param family column family name of the new cell
+ * @param qualifier qualifier for the new cell
+ * @param ts timestamp of the new cell
+ * @param newValue value of the new cell
+ * @param tags tags in the new cell
+ * @return cell
+ * @throws IOException while creating the cell.
+ */
+ public static Cell createNewCell(byte[] row, byte[] family, byte[] qualifier,
+ long ts, byte[] newValue, byte[] tags) throws IOException {
+ return CellUtil.createCell(row, family, qualifier, ts, KeyValue.Type.Put,
+ newValue, tags);
+ }
+
+ /**
+ * returns app id from the list of tags.
+ *
+ * @param tags cell tags to be looked into
+ * @return App Id as the AggregationCompactionDimension
+ */
+ public static String getAggregationCompactionDimension(List<Tag> tags) {
+ String appId = null;
+ for (Tag t : tags) {
+ if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t
+ .getType()) {
+ appId = Bytes.toString(t.getValue());
+ return appId;
+ }
+ }
+ return appId;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
index 7238efa..288046c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
@@ -33,7 +33,7 @@ public class TimestampGenerator {
* if this is changed, then reading cell timestamps written with older
* multiplier value will not work
*/
- public static final long TS_MULTIPLIER = 1000L;
+ public static final long TS_MULTIPLIER = 1000000L;
private final AtomicLong lastTimestamp = new AtomicLong();
@@ -74,13 +74,14 @@ public class TimestampGenerator {
}
/**
- * returns a timestamp multiplied with TS_MULTIPLIER and last few digits of
- * application id
+ * Returns a timestamp multiplied with TS_MULTIPLIER and last few digits of
+ * application id.
*
* Unlikely scenario of generating a timestamp that is a duplicate: If more
- * than a 1000 concurrent apps are running in one flow run AND write to same
- * column at the same time, then say appId of 1001 will overlap with appId of
- * 001 and there may be collisions for that flow run's specific column.
+ * than a 1M concurrent apps are running in one flow run AND write to same
+ * column at the same time, then say appId of 1M and 1 will overlap
+ * with appId of 001 and there may be collisions for that flow run's
+ * specific column.
*
* @param incomingTS Timestamp to be converted.
* @param appId Application Id.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
index 6240e81..40cdd2c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
@@ -21,19 +21,19 @@ import org.apache.hadoop.hbase.util.Bytes;
/**
* Identifies the attributes to be set for puts into the {@link FlowRunTable}.
- * The numbers used for tagType are prime numbers
+ * The numbers used for tagType are prime numbers.
*/
public enum AggregationOperation {
/**
* When the flow was started.
*/
- MIN((byte) 71),
+ GLOBAL_MIN((byte) 71),
/**
* When it ended.
*/
- MAX((byte) 73),
+ GLOBAL_MAX((byte) 73),
/**
* The metrics of the flow.
@@ -46,9 +46,16 @@ public enum AggregationOperation {
SUM_FINAL((byte) 83),
/**
- * compact.
+ * Min value as per the latest timestamp
+ * seen for a given app.
*/
- COMPACT((byte) 89);
+ LATEST_MIN((byte) 89),
+
+ /**
+ * Max value as per the latest timestamp
+ * seen for a given app.
+ */
+ LATEST_MAX((byte) 97);
private byte tagType;
private byte[] inBytes;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
index 148a37f..d50bb16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumn.java
@@ -41,14 +41,14 @@ public enum FlowRunColumn implements Column<FlowRunTable> {
* application start times.
*/
MIN_START_TIME(FlowRunColumnFamily.INFO, "min_start_time",
- AggregationOperation.MIN, LongConverter.getInstance()),
+ AggregationOperation.GLOBAL_MIN, LongConverter.getInstance()),
/**
* When the flow ended. This is the maximum of currently known application end
* times.
*/
MAX_END_TIME(FlowRunColumnFamily.INFO, "max_end_time",
- AggregationOperation.MAX, LongConverter.getInstance()),
+ AggregationOperation.GLOBAL_MAX, LongConverter.getInstance()),
/**
* The version of the flow that this flow belongs to.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
index 3d7c40e..fa94aae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
@@ -40,7 +40,7 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
/**
* To store flow run info values.
*/
- METRIC(FlowRunColumnFamily.INFO, "m", AggregationOperation.SUM,
+ METRIC(FlowRunColumnFamily.INFO, "m", null,
LongConverter.getInstance());
private final ColumnHelper<FlowRunTable> column;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
index 9698f06..450640a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -40,7 +40,12 @@ import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
import org.apache.hadoop.hbase.coprocessor.ObserverContext;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
+import org.apache.hadoop.hbase.regionserver.ScanType;
+import org.apache.hadoop.hbase.regionserver.Store;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
@@ -51,7 +56,6 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGen
*/
public class FlowRunCoprocessor extends BaseRegionObserver {
- @SuppressWarnings("unused")
private static final Log LOG = LogFactory.getLog(FlowRunCoprocessor.class);
private HRegion region;
@@ -160,8 +164,8 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
scan.setMaxVersions();
RegionScanner scanner = null;
try {
- scanner = new FlowScanner(region, scan.getBatch(),
- region.getScanner(scan));
+ scanner = new FlowScanner(e.getEnvironment(), scan.getBatch(),
+ region.getScanner(scan), FlowScannerOperation.READ);
scanner.next(results);
e.bypass();
} finally {
@@ -209,6 +213,64 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
public RegionScanner postScannerOpen(
ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
RegionScanner scanner) throws IOException {
- return new FlowScanner(region, scan.getBatch(), scanner);
+ return new FlowScanner(e.getEnvironment(), scan.getBatch(),
+ scanner, FlowScannerOperation.READ);
+ }
+
+ @Override
+ public InternalScanner preFlush(
+ ObserverContext<RegionCoprocessorEnvironment> c, Store store,
+ InternalScanner scanner) throws IOException {
+ if (LOG.isDebugEnabled()) {
+ if (store != null) {
+ LOG.debug("preFlush store = " + store.getColumnFamilyName()
+ + " flushableSize=" + store.getFlushableSize()
+ + " flushedCellsCount=" + store.getFlushedCellsCount()
+ + " compactedCellsCount=" + store.getCompactedCellsCount()
+ + " majorCompactedCellsCount="
+ + store.getMajorCompactedCellsCount() + " memstoreFlushSize="
+ + store.getMemstoreFlushSize() + " memstoreSize="
+ + store.getMemStoreSize() + " size=" + store.getSize()
+ + " storeFilesCount=" + store.getStorefilesCount());
+ }
+ }
+ return new FlowScanner(c.getEnvironment(), -1, scanner,
+ FlowScannerOperation.FLUSH);
+ }
+
+ @Override
+ public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c,
+ Store store, StoreFile resultFile) {
+ if (LOG.isDebugEnabled()) {
+ if (store != null) {
+ LOG.debug("postFlush store = " + store.getColumnFamilyName()
+ + " flushableSize=" + store.getFlushableSize()
+ + " flushedCellsCount=" + store.getFlushedCellsCount()
+ + " compactedCellsCount=" + store.getCompactedCellsCount()
+ + " majorCompactedCellsCount="
+ + store.getMajorCompactedCellsCount() + " memstoreFlushSize="
+ + store.getMemstoreFlushSize() + " memstoreSize="
+ + store.getMemStoreSize() + " size=" + store.getSize()
+ + " storeFilesCount=" + store.getStorefilesCount());
+ }
+ }
+ }
+
+ @Override
+ public InternalScanner preCompact(
+ ObserverContext<RegionCoprocessorEnvironment> e, Store store,
+ InternalScanner scanner, ScanType scanType, CompactionRequest request)
+ throws IOException {
+
+ FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION;
+ if (request != null) {
+ requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION
+ : FlowScannerOperation.MINOR_COMPACTION);
+ LOG.info("Compactionrequest= " + request.toString() + " "
+ + requestOp.toString() + " RegionName="
+ + e.getEnvironment().getRegion().getRegionNameAsString());
+ }
+
+ return new FlowScanner(e.getEnvironment(), -1, scanner, requestOp);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
index 0585dc9..eac8f05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -113,4 +113,20 @@ public class FlowRunRowKey {
TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
return new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
}
+
+ /**
+ * returns the Flow Key as a verbose String output.
+ * @return String
+ */
+ @Override
+ public String toString() {
+ StringBuilder flowKeyStr = new StringBuilder();
+ flowKeyStr.append("{clusterId=" + clusterId);
+ flowKeyStr.append(" userId=" + userId);
+ flowKeyStr.append(" flowName=" + flowName);
+ flowKeyStr.append(" flowRunId=");
+ flowKeyStr.append(flowRunId);
+ flowKeyStr.append("}");
+ return flowKeyStr.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
index 6fefd15..6baea37 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -29,20 +29,26 @@ import java.util.TreeSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Bytes.ByteArrayComparator;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.GenericConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.NumericValueConverter;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ValueConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* Invoked via the coprocessor when a Get or a Scan is issued for flow run
@@ -55,23 +61,42 @@ class FlowScanner implements RegionScanner, Closeable {
private static final Log LOG = LogFactory.getLog(FlowScanner.class);
+ /**
+ * use a special application id to represent the flow id this is needed since
+ * TimestampGenerator parses the app id to generate a cell timestamp.
+ */
+ private static final String FLOW_APP_ID = "application_00000000000_0000";
+
private final HRegion region;
private final InternalScanner flowRunScanner;
- private RegionScanner regionScanner;
private final int limit;
+ private final long appFinalValueRetentionThreshold;
+ private RegionScanner regionScanner;
private boolean hasMore;
private byte[] currentRow;
private List<Cell> availableCells = new ArrayList<>();
private int currentIndex;
+ private FlowScannerOperation action = FlowScannerOperation.READ;
- FlowScanner(HRegion region, int limit, InternalScanner internalScanner) {
- this.region = region;
+ FlowScanner(RegionCoprocessorEnvironment env, int limit,
+ InternalScanner internalScanner, FlowScannerOperation action) {
this.limit = limit;
this.flowRunScanner = internalScanner;
if (internalScanner instanceof RegionScanner) {
this.regionScanner = (RegionScanner) internalScanner;
}
- // TODO: note if it's compaction/flush
+ this.action = action;
+ if (env == null) {
+ this.appFinalValueRetentionThreshold =
+ YarnConfiguration.DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD;
+ this.region = null;
+ } else {
+ this.region = env.getRegion();
+ Configuration hbaseConf = env.getConfiguration();
+ this.appFinalValueRetentionThreshold = hbaseConf.getLong(
+ YarnConfiguration.APP_FINAL_VALUE_RETENTION_THRESHOLD,
+ YarnConfiguration.DEFAULT_APP_FINAL_VALUE_RETENTION_THRESHOLD);
+ }
}
/*
@@ -104,17 +129,6 @@ class FlowScanner implements RegionScanner, Closeable {
return nextInternal(cells, cellLimit);
}
- private String getAggregationCompactionDimension(List<Tag> tags) {
- String appId = null;
- for (Tag t : tags) {
- if (AggregationCompactionDimension.APPLICATION_ID.getTagType() == t
- .getType()) {
- appId = Bytes.toString(t.getValue());
- }
- }
- return appId;
- }
-
/**
* Get value converter associated with a column or a column prefix. If nothing
* matches, generic converter is returned.
@@ -165,6 +179,7 @@ class FlowScanner implements RegionScanner, Closeable {
* @return true if next row is available for the scanner, false otherwise
* @throws IOException
*/
+ @SuppressWarnings("deprecation")
private boolean nextInternal(List<Cell> cells, int cellLimit)
throws IOException {
Cell cell = null;
@@ -183,14 +198,18 @@ class FlowScanner implements RegionScanner, Closeable {
SortedSet<Cell> currentColumnCells = new TreeSet<>(KeyValue.COMPARATOR);
Set<String> alreadySeenAggDim = new HashSet<>();
int addedCnt = 0;
+ long currentTimestamp = System.currentTimeMillis();
ValueConverter converter = null;
- while (((cell = peekAtNextCell(cellLimit)) != null)
- && (cellLimit <= 0 || addedCnt < cellLimit)) {
+ while (cellLimit <= 0 || addedCnt < cellLimit) {
+ cell = peekAtNextCell(cellLimit);
+ if (cell == null) {
+ break;
+ }
byte[] newColumnQualifier = CellUtil.cloneQualifier(cell);
if (comp.compare(currentColumnQualifier, newColumnQualifier) != 0) {
if (converter != null && isNumericConverter(converter)) {
addedCnt += emitCells(cells, currentColumnCells, currentAggOp,
- (NumericValueConverter)converter);
+ (NumericValueConverter)converter, currentTimestamp);
}
resetState(currentColumnCells, alreadySeenAggDim);
currentColumnQualifier = newColumnQualifier;
@@ -207,8 +226,17 @@ class FlowScanner implements RegionScanner, Closeable {
nextCell(cellLimit);
}
if (!currentColumnCells.isEmpty()) {
- emitCells(cells, currentColumnCells, currentAggOp,
- (NumericValueConverter)converter);
+ addedCnt += emitCells(cells, currentColumnCells, currentAggOp,
+ (NumericValueConverter)converter, currentTimestamp);
+ if (LOG.isDebugEnabled()) {
+ if (addedCnt > 0) {
+ LOG.debug("emitted cells. " + addedCnt + " for " + this.action
+ + " rowKey="
+ + FlowRunRowKey.parseRowKey(cells.get(0).getRow()).toString());
+ } else {
+ LOG.debug("emitted no cells for " + this.action);
+ }
+ }
}
return hasMore();
}
@@ -247,7 +275,7 @@ class FlowScanner implements RegionScanner, Closeable {
}
switch (currentAggOp) {
- case MIN:
+ case GLOBAL_MIN:
if (currentColumnCells.size() == 0) {
currentColumnCells.add(cell);
} else {
@@ -260,7 +288,7 @@ class FlowScanner implements RegionScanner, Closeable {
}
}
break;
- case MAX:
+ case GLOBAL_MAX:
if (currentColumnCells.size() == 0) {
currentColumnCells.add(cell);
} else {
@@ -275,16 +303,32 @@ class FlowScanner implements RegionScanner, Closeable {
break;
case SUM:
case SUM_FINAL:
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("In collect cells "
+ + " FlowSannerOperation="
+ + this.action
+ + " currentAggOp="
+ + currentAggOp
+ + " cell qualifier="
+ + Bytes.toString(CellUtil.cloneQualifier(cell))
+ + " cell value= "
+ + (Number) converter.decodeValue(CellUtil.cloneValue(cell))
+ + " timestamp=" + cell.getTimestamp());
+ }
+
// only if this app has not been seen yet, add to current column cells
List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
cell.getTagsLength());
- String aggDim = getAggregationCompactionDimension(tags);
-
- // If this agg dimension has already been seen, since they show up in
- // sorted order, we drop the rest which are older. In other words, this
- // cell is older than previously seen cells for that agg dim.
+ String aggDim = TimelineStorageUtils
+ .getAggregationCompactionDimension(tags);
if (!alreadySeenAggDim.contains(aggDim)) {
- // Not seen this agg dim, hence consider this cell in our working set
+ // if this agg dimension has already been seen,
+ // since they show up in sorted order
+ // we drop the rest which are older
+ // in other words, this cell is older than previously seen cells
+ // for that agg dim
+ // but when this agg dim is not seen,
+ // consider this cell in our working set
currentColumnCells.add(cell);
alreadySeenAggDim.add(aggDim);
}
@@ -300,8 +344,8 @@ class FlowScanner implements RegionScanner, Closeable {
* parameter.
*/
private int emitCells(List<Cell> cells, SortedSet<Cell> currentColumnCells,
- AggregationOperation currentAggOp, NumericValueConverter converter)
- throws IOException {
+ AggregationOperation currentAggOp, NumericValueConverter converter,
+ long currentTimestamp) throws IOException {
if ((currentColumnCells == null) || (currentColumnCells.size() == 0)) {
return 0;
}
@@ -309,17 +353,36 @@ class FlowScanner implements RegionScanner, Closeable {
cells.addAll(currentColumnCells);
return currentColumnCells.size();
}
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("In emitCells " + this.action + " currentColumnCells size= "
+ + currentColumnCells.size() + " currentAggOp" + currentAggOp);
+ }
switch (currentAggOp) {
- case MIN:
- case MAX:
+ case GLOBAL_MIN:
+ case GLOBAL_MAX:
cells.addAll(currentColumnCells);
return currentColumnCells.size();
case SUM:
case SUM_FINAL:
- Cell sumCell = processSummation(currentColumnCells, converter);
- cells.add(sumCell);
- return 1;
+ switch (action) {
+ case FLUSH:
+ case MINOR_COMPACTION:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ case READ:
+ Cell sumCell = processSummation(currentColumnCells, converter);
+ cells.add(sumCell);
+ return 1;
+ case MAJOR_COMPACTION:
+ List<Cell> finalCells = processSummationMajorCompaction(
+ currentColumnCells, converter, currentTimestamp);
+ cells.addAll(finalCells);
+ return finalCells.size();
+ default:
+ cells.addAll(currentColumnCells);
+ return currentColumnCells.size();
+ }
default:
cells.addAll(currentColumnCells);
return currentColumnCells.size();
@@ -349,10 +412,122 @@ class FlowScanner implements RegionScanner, Closeable {
sum = converter.add(sum, currentValue);
}
byte[] sumBytes = converter.encodeValue(sum);
- Cell sumCell = createNewCell(mostRecentCell, sumBytes);
+ Cell sumCell = TimelineStorageUtils.createNewCell(mostRecentCell, sumBytes);
return sumCell;
}
+
+ /**
+ * Returns a list of cells that contains
+ *
+ * A) the latest cells for applications that haven't finished yet
+ * B) summation
+ * for the flow, based on applications that have completed and are older than
+ * a certain time
+ *
+ * The new cell created has the timestamp of the most recent metric cell. The
+ * sum of a metric for a flow run is the summation at the point of the last
+ * metric update in that flow till that time.
+ */
+ @VisibleForTesting
+ List<Cell> processSummationMajorCompaction(
+ SortedSet<Cell> currentColumnCells, NumericValueConverter converter,
+ long currentTimestamp)
+ throws IOException {
+ Number sum = 0;
+ Number currentValue = 0;
+ long ts = 0L;
+ boolean summationDone = false;
+ List<Cell> finalCells = new ArrayList<Cell>();
+ if (currentColumnCells == null) {
+ return finalCells;
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("In processSummationMajorCompaction,"
+ + " will drop cells older than " + currentTimestamp
+ + " CurrentColumnCells size=" + currentColumnCells.size());
+ }
+
+ for (Cell cell : currentColumnCells) {
+ AggregationOperation cellAggOp = getCurrentAggOp(cell);
+ // if this is the existing flow sum cell
+ List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
+ cell.getTagsLength());
+ String appId = TimelineStorageUtils
+ .getAggregationCompactionDimension(tags);
+ if (appId == FLOW_APP_ID) {
+ sum = converter.add(sum, currentValue);
+ summationDone = true;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("reading flow app id sum=" + sum);
+ }
+ } else {
+ currentValue = (Number) converter.decodeValue(CellUtil
+ .cloneValue(cell));
+ // read the timestamp truncated by the generator
+ ts = TimestampGenerator.getTruncatedTimestamp(cell.getTimestamp());
+ if ((cellAggOp == AggregationOperation.SUM_FINAL)
+ && ((ts + this.appFinalValueRetentionThreshold)
+ < currentTimestamp)) {
+ sum = converter.add(sum, currentValue);
+ summationDone = true;
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("MAJOR COMPACTION loop sum= " + sum
+ + " discarding now: " + " qualifier="
+ + Bytes.toString(CellUtil.cloneQualifier(cell)) + " value="
+ + (Number) converter.decodeValue(CellUtil.cloneValue(cell))
+ + " timestamp=" + cell.getTimestamp() + " " + this.action);
+ }
+ } else {
+ // not a final value but it's the latest cell for this app
+ // so include this cell in the list of cells to write back
+ finalCells.add(cell);
+ }
+ }
+ }
+ if (summationDone) {
+ Cell anyCell = currentColumnCells.first();
+ List<Tag> tags = new ArrayList<Tag>();
+ Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ Bytes.toBytes(FLOW_APP_ID));
+ tags.add(t);
+ t = new Tag(AggregationCompactionDimension.APPLICATION_ID.getTagType(),
+ Bytes.toBytes(FLOW_APP_ID));
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ Cell sumCell = TimelineStorageUtils.createNewCell(
+ CellUtil.cloneRow(anyCell),
+ CellUtil.cloneFamily(anyCell),
+ CellUtil.cloneQualifier(anyCell),
+ TimestampGenerator.getSupplementedTimestamp(
+ System.currentTimeMillis(), FLOW_APP_ID),
+ converter.encodeValue(sum), tagByteArray);
+ finalCells.add(sumCell);
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("MAJOR COMPACTION final sum= " + sum + " for "
+ + Bytes.toString(CellUtil.cloneQualifier(sumCell))
+ + " " + this.action);
+ }
+ LOG.info("After major compaction for qualifier="
+ + Bytes.toString(CellUtil.cloneQualifier(sumCell))
+ + " with currentColumnCells.size="
+ + currentColumnCells.size()
+ + " returning finalCells.size=" + finalCells.size()
+ + " with sum=" + sum.longValue()
+ + " with cell timestamp " + sumCell.getTimestamp());
+ } else {
+ String qualifier = "";
+ LOG.info("After major compaction for qualifier=" + qualifier
+ + " with currentColumnCells.size="
+ + currentColumnCells.size()
+ + " returning finalCells.size=" + finalCells.size()
+ + " with zero sum="
+ + sum.longValue());
+ }
+ return finalCells;
+ }
+
/**
* Determines which cell is to be returned based on the values in each cell
* and the comparison operation MIN or MAX.
@@ -375,7 +550,7 @@ class FlowScanner implements RegionScanner, Closeable {
Number currentCellValue = (Number) converter.decodeValue(CellUtil
.cloneValue(currentCell));
switch (currentAggOp) {
- case MIN:
+ case GLOBAL_MIN:
if (converter.compare(
currentCellValue, previouslyChosenCellValue) < 0) {
// new value is minimum, hence return this cell
@@ -384,7 +559,7 @@ class FlowScanner implements RegionScanner, Closeable {
// previously chosen value is miniumum, hence return previous min cell
return previouslyChosenCell;
}
- case MAX:
+ case GLOBAL_MAX:
if (converter.compare(
currentCellValue, previouslyChosenCellValue) > 0) {
// new value is max, hence return this cell
@@ -402,16 +577,13 @@ class FlowScanner implements RegionScanner, Closeable {
}
}
- private Cell createNewCell(Cell origCell, byte[] newValue)
- throws IOException {
- return CellUtil.createCell(CellUtil.cloneRow(origCell),
- CellUtil.cloneFamily(origCell), CellUtil.cloneQualifier(origCell),
- origCell.getTimestamp(), KeyValue.Type.Put.getCode(), newValue);
- }
-
@Override
public void close() throws IOException {
- flowRunScanner.close();
+ if (flowRunScanner != null) {
+ flowRunScanner.close();
+ } else {
+ LOG.warn("scanner close called but scanner is null");
+ }
}
/**
@@ -423,8 +595,6 @@ class FlowScanner implements RegionScanner, Closeable {
/**
* Returns whether or not the underlying scanner has more rows.
- *
- * @return true, if there are more cells to return, false otherwise.
*/
public boolean hasMore() {
return currentIndex < availableCells.size() ? true : hasMore;
@@ -440,8 +610,7 @@ class FlowScanner implements RegionScanner, Closeable {
* fetched by the wrapped scanner
* @return the next available cell or null if no more cells are available for
* the current row
- * @throws IOException if any problem is encountered while grabbing the next
- * cell.
+ * @throws IOException
*/
public Cell nextCell(int cellLimit) throws IOException {
Cell cell = peekAtNextCell(cellLimit);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
new file mode 100644
index 0000000..73c666f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScannerOperation.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+
+/**
+ * Identifies the scanner operation on the {@link FlowRunTable}.
+ */
+public enum FlowScannerOperation {
+
+ /**
+ * If the scanner is opened for reading
+ * during preGet or preScan.
+ */
+ READ,
+
+ /**
+ * If the scanner is opened during preFlush.
+ */
+ FLUSH,
+
+ /**
+ * If the scanner is opened during minor Compaction.
+ */
+ MINOR_COMPACTION,
+
+ /**
+ * If the scanner is opened during major Compaction.
+ */
+ MAJOR_COMPACTION
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index d45df57..9793ce6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
-
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -29,17 +28,60 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.conf.Configuration;
/**
* Generates the data/entities for the FlowRun and FlowActivity Tables
*/
class TestFlowDataGenerator {
- private final static String metric1 = "MAP_SLOT_MILLIS";
- private final static String metric2 = "HDFS_BYTES_READ";
+ private static final String metric1 = "MAP_SLOT_MILLIS";
+ private static final String metric2 = "HDFS_BYTES_READ";
+ public static final long END_TS_INCR = 10000L;
+
+ static TimelineEntity getEntityMetricsApp1(long insertTs, Configuration c1) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunMetrics_test";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId(metric1);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = insertTs;
- static TimelineEntity getEntityMetricsApp1() {
+ for (int k=1; k< 100 ; k++) {
+ metricValues.put(ts - k*200000, 20L);
+ }
+ metricValues.put(ts - 80000, 40L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId(metric2);
+ metricValues = new HashMap<Long, Number>();
+ ts = System.currentTimeMillis();
+ for (int k=1; k< 100 ; k++) {
+ metricValues.put(ts - k*100000, 31L);
+ }
+
+ metricValues.put(ts - 80000, 57L);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+
+ entity.addMetrics(metrics);
+ return entity;
+ }
+
+
+ static TimelineEntity getEntityMetricsApp1Complete(long insertTs, Configuration c1) {
TimelineEntity entity = new TimelineEntity();
String id = "flowRunMetrics_test";
String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -53,7 +95,48 @@ class TestFlowDataGenerator {
TimelineMetric m1 = new TimelineMetric();
m1.setId(metric1);
Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
+ long ts = insertTs;
+
+ metricValues.put(ts - 80000, 40L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId(metric2);
+ metricValues = new HashMap<Long, Number>();
+ ts = insertTs;
+ metricValues.put(ts - 80000, 57L);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+
+ entity.addMetrics(metrics);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ event.setTimestamp(insertTs);
+ event.addInfo("done", "insertTs=" + insertTs);
+ entity.addEvent(event);
+ return entity;
+ }
+
+
+ static TimelineEntity getEntityMetricsApp1(long insertTs) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunMetrics_test";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId(metric1);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ long ts = insertTs;
metricValues.put(ts - 100000, 2L);
metricValues.put(ts - 80000, 40L);
m1.setType(Type.TIME_SERIES);
@@ -63,7 +146,7 @@ class TestFlowDataGenerator {
TimelineMetric m2 = new TimelineMetric();
m2.setId(metric2);
metricValues = new HashMap<Long, Number>();
- ts = System.currentTimeMillis();
+ ts = insertTs;
metricValues.put(ts - 100000, 31L);
metricValues.put(ts - 80000, 57L);
m2.setType(Type.TIME_SERIES);
@@ -74,7 +157,8 @@ class TestFlowDataGenerator {
return entity;
}
- static TimelineEntity getEntityMetricsApp2() {
+
+ static TimelineEntity getEntityMetricsApp2(long insertTs) {
TimelineEntity entity = new TimelineEntity();
String id = "flowRunMetrics_test";
String type = TimelineEntityType.YARN_APPLICATION.toString();
@@ -87,7 +171,7 @@ class TestFlowDataGenerator {
TimelineMetric m1 = new TimelineMetric();
m1.setId(metric1);
Map<Long, Number> metricValues = new HashMap<Long, Number>();
- long ts = System.currentTimeMillis();
+ long ts = insertTs;
metricValues.put(ts - 100000, 5L);
metricValues.put(ts - 80000, 101L);
m1.setType(Type.TIME_SERIES);
@@ -140,6 +224,55 @@ class TestFlowDataGenerator {
return entity;
}
+ static TimelineEntity getAFullEntity(long ts, long endTs) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunFullEntity";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(ts);
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId(metric1);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 120000, 100000000L);
+ metricValues.put(ts - 100000, 200000000L);
+ metricValues.put(ts - 80000, 300000000L);
+ metricValues.put(ts - 60000, 400000000L);
+ metricValues.put(ts - 40000, 50000000000L);
+ metricValues.put(ts - 20000, 60000000000L);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId(metric2);
+ metricValues = new HashMap<Long, Number>();
+ metricValues.put(ts - 900000, 31L);
+ metricValues.put(ts - 30000, 57L);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+ entity.addMetrics(metrics);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event.setTimestamp(ts);
+ String expKey = "foo_event";
+ Object expVal = "test";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+
+ event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ long expTs = ts + 21600000;// start time + 6hrs
+ event.setTimestamp(expTs);
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
+
+ return entity;
+ }
+
static TimelineEntity getEntityGreaterStartTime(long startTs) {
TimelineEntity entity = new TimelineEntity();
entity.setCreatedTime(startTs);
@@ -184,6 +317,34 @@ class TestFlowDataGenerator {
return entity;
}
+ static TimelineEntity getMinFlushEntity(long startTs) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunHelloFlushEntityMin";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(startTs);
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ event.setTimestamp(startTs);
+ entity.addEvent(event);
+ return entity;
+ }
+
+ static TimelineEntity getMaxFlushEntity(long startTs) {
+ TimelineEntity entity = new TimelineEntity();
+ String id = "flowRunHelloFlushEntityMax";
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ entity.setId(id);
+ entity.setType(type);
+ entity.setCreatedTime(startTs);
+
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ event.setTimestamp(startTs + END_TS_INCR);
+ entity.addEvent(event);
+ return entity;
+ }
static TimelineEntity getFlowApp1(long appCreatedTime) {
TimelineEntity entity = new TimelineEntity();
@@ -203,5 +364,4 @@ class TestFlowDataGenerator {
return entity;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index b234bfd..f04dd48 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilte
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -216,7 +216,8 @@ public class TestHBaseStorageFlowRun {
long runid = 1002345678919L;
TimelineEntities te = new TimelineEntities();
- TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1();
+ TimelineEntity entityApp1 = TestFlowDataGenerator
+ .getEntityMetricsApp1(System.currentTimeMillis());
te.addEntity(entityApp1);
HBaseTimelineWriterImpl hbi = null;
@@ -228,7 +229,8 @@ public class TestHBaseStorageFlowRun {
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
// write another application with same metric to this flow
te = new TimelineEntities();
- TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2();
+ TimelineEntity entityApp2 = TestFlowDataGenerator
+ .getEntityMetricsApp2(System.currentTimeMillis());
te.addEntity(entityApp2);
appName = "application_11111111111111_2222";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -323,7 +325,8 @@ public class TestHBaseStorageFlowRun {
long runid = 1002345678919L;
TimelineEntities te = new TimelineEntities();
- TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1();
+ TimelineEntity entityApp1 = TestFlowDataGenerator
+ .getEntityMetricsApp1(System.currentTimeMillis());
te.addEntity(entityApp1);
HBaseTimelineWriterImpl hbi = null;
@@ -335,7 +338,8 @@ public class TestHBaseStorageFlowRun {
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
// write another application with same metric to this flow
te = new TimelineEntities();
- TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2();
+ TimelineEntity entityApp2 = TestFlowDataGenerator
+ .getEntityMetricsApp2(System.currentTimeMillis());
te.addEntity(entityApp2);
appName = "application_11111111111111_2222";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -420,7 +424,8 @@ public class TestHBaseStorageFlowRun {
long runid = 1002345678919L;
TimelineEntities te = new TimelineEntities();
- TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1();
+ TimelineEntity entityApp1 = TestFlowDataGenerator
+ .getEntityMetricsApp1(System.currentTimeMillis());
te.addEntity(entityApp1);
HBaseTimelineWriterImpl hbi = null;
@@ -432,7 +437,8 @@ public class TestHBaseStorageFlowRun {
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
// write another application with same metric to this flow
te = new TimelineEntities();
- TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2();
+ TimelineEntity entityApp2 = TestFlowDataGenerator
+ .getEntityMetricsApp2(System.currentTimeMillis());
te.addEntity(entityApp2);
appName = "application_11111111111111_2222";
hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
@@ -494,6 +500,98 @@ public class TestHBaseStorageFlowRun {
}
}
+ @Test
+ public void testWriteFlowRunFlush() throws Exception {
+ String cluster = "atestFlushFlowRun_cluster1";
+ String user = "atestFlushFlowRun__user1";
+ String flow = "atestFlushFlowRun_flow_name";
+ String flowVersion = "AF1021C19F1351";
+ long runid = 1449526652000L;
+
+ int start = 10;
+ int count = 20000;
+ int appIdSuffix = 1;
+ HBaseTimelineWriterImpl hbi = null;
+ long insertTs = 1449796654827L - count;
+ long minTS = insertTs + 1;
+ long startTs = insertTs;
+ Configuration c1 = util.getConfiguration();
+ TimelineEntities te1 = null;
+ TimelineEntity entityApp1 = null;
+ TimelineEntity entityApp2 = null;
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+
+ for (int i = start; i < count; i++) {
+ String appName = "application_1060350000000_" + appIdSuffix;
+ insertTs++;
+ te1 = new TimelineEntities();
+ entityApp1 = TestFlowDataGenerator.getMinFlushEntity(insertTs);
+ te1.addEntity(entityApp1);
+ entityApp2 = TestFlowDataGenerator.getMaxFlushEntity(insertTs);
+ te1.addEntity(entityApp2);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
+ Thread.sleep(1);
+
+ appName = "application_1001199480000_7" + appIdSuffix;
+ insertTs++;
+ appIdSuffix++;
+ te1 = new TimelineEntities();
+ entityApp1 = TestFlowDataGenerator.getMinFlushEntity(insertTs);
+ te1.addEntity(entityApp1);
+ entityApp2 = TestFlowDataGenerator.getMaxFlushEntity(insertTs);
+ te1.addEntity(entityApp2);
+
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
+ if (i % 1000 == 0) {
+ hbi.flush();
+ checkMinMaxFlush(c1, minTS, startTs, count, cluster, user, flow,
+ runid, false);
+ }
+ }
+ } finally {
+ hbi.flush();
+ hbi.close();
+ checkMinMaxFlush(c1, minTS, startTs, count, cluster, user, flow, runid,
+ true);
+ }
+ }
+
+ private void checkMinMaxFlush(Configuration c1, long minTS, long startTs,
+ int count, String cluster, String user, String flow, long runid,
+ boolean checkMax) throws IOException {
+ Connection conn = ConnectionFactory.createConnection(c1);
+ // check in flow run table
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
+ // scan the table and see that we get back the right min and max
+ // timestamps
+ byte[] startRow = FlowRunRowKey.getRowKey(cluster, user, flow, runid);
+ Get g = new Get(startRow);
+ g.addColumn(FlowRunColumnFamily.INFO.getBytes(),
+ FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes());
+ g.addColumn(FlowRunColumnFamily.INFO.getBytes(),
+ FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes());
+
+ Result r1 = table1.get(g);
+ assertNotNull(r1);
+ assertTrue(!r1.isEmpty());
+ Map<byte[], byte[]> values = r1.getFamilyMap(FlowRunColumnFamily.INFO
+ .getBytes());
+ int start = 10;
+ assertEquals(2, r1.size());
+ long starttime = Bytes.toLong(values
+ .get(FlowRunColumn.MIN_START_TIME.getColumnQualifierBytes()));
+ assertEquals(minTS, starttime);
+ if (checkMax) {
+ assertEquals(startTs + 2 * (count - start)
+ + TestFlowDataGenerator.END_TS_INCR,
+ Bytes.toLong(values
+ .get(FlowRunColumn.MAX_END_TIME.getColumnQualifierBytes())));
+ }
+ }
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters
in TimelineReader (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 4e07ecf..6b57ec4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -52,10 +53,14 @@ import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
@@ -75,9 +80,6 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.ImmutableSet;
-
/**
* Various tests to test writing entities to HBase and reading them back from
* it.
@@ -113,30 +115,29 @@ public class TestHBaseTimelineStorage {
String id = "application_1111111111_2222";
entity.setId(id);
entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
- Long cTime = 1425016501000L;
+ Long cTime = 1425016502000L;
entity.setCreatedTime(cTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap = new HashMap<String, Object>();
- infoMap.put("infoMapKey1", "infoMapValue1");
- infoMap.put("infoMapKey2", 10);
+ infoMap.put("infoMapKey1", "infoMapValue2");
+ infoMap.put("infoMapKey2", 20);
+ infoMap.put("infoMapKey3", 85.85);
entity.addInfo(infoMap);
// add the isRelatedToEntity info
- String key = "task";
- String value = "is_related_to_entity_id_here";
Set<String> isRelatedToSet = new HashSet<String>();
- isRelatedToSet.add(value);
+ isRelatedToSet.add("relatedto1");
Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- isRelatedTo.put(key, isRelatedToSet);
+ isRelatedTo.put("task", isRelatedToSet);
entity.setIsRelatedToEntities(isRelatedTo);
// add the relatesTo info
- key = "container";
- value = "relates_to_entity_id_here";
Set<String> relatesToSet = new HashSet<String>();
- relatesToSet.add(value);
- value = "relates_to_entity_id_here_Second";
- relatesToSet.add(value);
+ relatesToSet.add("relatesto1");
+ relatesToSet.add("relatesto3");
Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- relatesTo.put(key, relatesToSet);
+ relatesTo.put("container", relatesToSet);
+ Set<String> relatesToSet11 = new HashSet<String>();
+ relatesToSet11.add("relatesto4");
+ relatesTo.put("container1", relatesToSet11);
entity.setRelatesToEntities(relatesTo);
// add some config entries
Map<String, String> conf = new HashMap<String, String>();
@@ -166,8 +167,8 @@ public class TestHBaseTimelineStorage {
metrics.add(m12);
entity.addMetrics(metrics);
TimelineEvent event = new TimelineEvent();
- event.setId("event1");
- event.setTimestamp(ts - 2000);
+ event.setId("start_event");
+ event.setTimestamp(ts);
entity.addEvent(event);
te.addEntity(entity);
@@ -176,7 +177,7 @@ public class TestHBaseTimelineStorage {
String id1 = "application_1111111111_3333";
entity1.setId(id1);
entity1.setType(TimelineEntityType.YARN_APPLICATION.toString());
- entity1.setCreatedTime(cTime);
+ entity1.setCreatedTime(cTime + 20L);
// add the info map in Timeline Entity
Map<String, Object> infoMap1 = new HashMap<String, Object>();
@@ -185,23 +186,22 @@ public class TestHBaseTimelineStorage {
entity1.addInfo(infoMap1);
// add the isRelatedToEntity info
- String key1 = "task";
- String value1 = "is_related_to_entity_id_here";
Set<String> isRelatedToSet1 = new HashSet<String>();
- isRelatedToSet1.add(value1);
+ isRelatedToSet1.add("relatedto3");
+ isRelatedToSet1.add("relatedto5");
Map<String, Set<String>> isRelatedTo1 = new HashMap<String, Set<String>>();
- isRelatedTo1.put(key, isRelatedToSet1);
+ isRelatedTo1.put("task1", isRelatedToSet1);
+ Set<String> isRelatedToSet11 = new HashSet<String>();
+ isRelatedToSet11.add("relatedto4");
+ isRelatedTo1.put("task2", isRelatedToSet11);
entity1.setIsRelatedToEntities(isRelatedTo1);
// add the relatesTo info
- key1 = "container";
- value1 = "relates_to_entity_id_here";
Set<String> relatesToSet1 = new HashSet<String>();
- relatesToSet1.add(value1);
- value1 = "relates_to_entity_id_here_Second";
- relatesToSet1.add(value1);
+ relatesToSet1.add("relatesto1");
+ relatesToSet1.add("relatesto2");
Map<String, Set<String>> relatesTo1 = new HashMap<String, Set<String>>();
- relatesTo1.put(key1, relatesToSet1);
+ relatesTo1.put("container", relatesToSet1);
entity1.setRelatesToEntities(relatesTo1);
// add some config entries
@@ -226,6 +226,14 @@ public class TestHBaseTimelineStorage {
m2.setValues(metricValues1);
metrics1.add(m2);
entity1.addMetrics(metrics1);
+ TimelineEvent event11 = new TimelineEvent();
+ event11.setId("end_event");
+ event11.setTimestamp(ts);
+ entity1.addEvent(event11);
+ TimelineEvent event12 = new TimelineEvent();
+ event12.setId("update_event");
+ event12.setTimestamp(ts - 10);
+ entity1.addEvent(event12);
te1.addEntity(entity1);
TimelineEntities te2 = new TimelineEntities();
@@ -233,7 +241,22 @@ public class TestHBaseTimelineStorage {
String id2 = "application_1111111111_4444";
entity2.setId(id2);
entity2.setType(TimelineEntityType.YARN_APPLICATION.toString());
- entity2.setCreatedTime(cTime);
+ entity2.setCreatedTime(cTime + 40L);
+ TimelineEvent event21 = new TimelineEvent();
+ event21.setId("update_event");
+ event21.setTimestamp(ts - 20);
+ entity2.addEvent(event21);
+ Set<String> isRelatedToSet2 = new HashSet<String>();
+ isRelatedToSet2.add("relatedto3");
+ Map<String, Set<String>> isRelatedTo2 = new HashMap<String, Set<String>>();
+ isRelatedTo2.put("task1", isRelatedToSet2);
+ entity2.setIsRelatedToEntities(isRelatedTo2);
+ Map<String, Set<String>> relatesTo3 = new HashMap<String, Set<String>>();
+ Set<String> relatesToSet14 = new HashSet<String>();
+ relatesToSet14.add("relatesto7");
+ relatesTo3.put("container2", relatesToSet14);
+ entity2.setRelatesToEntities(relatesTo3);
+
te2.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
@@ -267,31 +290,30 @@ public class TestHBaseTimelineStorage {
String type = "world";
entity.setId(id);
entity.setType(type);
- Long cTime = 1425016501000L;
+ Long cTime = 1425016502000L;
entity.setCreatedTime(cTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap = new HashMap<String, Object>();
- infoMap.put("infoMapKey1", "infoMapValue1");
- infoMap.put("infoMapKey2", 10);
+ infoMap.put("infoMapKey1", "infoMapValue2");
+ infoMap.put("infoMapKey2", 20);
+ infoMap.put("infoMapKey3", 71.4);
entity.addInfo(infoMap);
// add the isRelatedToEntity info
- String key = "task";
- String value = "is_related_to_entity_id_here";
Set<String> isRelatedToSet = new HashSet<String>();
- isRelatedToSet.add(value);
+ isRelatedToSet.add("relatedto1");
Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- isRelatedTo.put(key, isRelatedToSet);
+ isRelatedTo.put("task", isRelatedToSet);
entity.setIsRelatedToEntities(isRelatedTo);
// add the relatesTo info
- key = "container";
- value = "relates_to_entity_id_here";
Set<String> relatesToSet = new HashSet<String>();
- relatesToSet.add(value);
- value = "relates_to_entity_id_here_Second";
- relatesToSet.add(value);
+ relatesToSet.add("relatesto1");
+ relatesToSet.add("relatesto3");
Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- relatesTo.put(key, relatesToSet);
+ relatesTo.put("container", relatesToSet);
+ Set<String> relatesToSet11 = new HashSet<String>();
+ relatesToSet11.add("relatesto4");
+ relatesTo.put("container1", relatesToSet11);
entity.setRelatesToEntities(relatesTo);
// add some config entries
@@ -312,7 +334,7 @@ public class TestHBaseTimelineStorage {
metricValues.put(ts - 80000, 300000000);
metricValues.put(ts - 60000, 400000000);
metricValues.put(ts - 40000, 50000000000L);
- metricValues.put(ts - 20000, 60000000000L);
+ metricValues.put(ts - 20000, 70000000000L);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
@@ -322,13 +344,17 @@ public class TestHBaseTimelineStorage {
m12.addValue(ts, 50);
metrics.add(m12);
entity.addMetrics(metrics);
+ TimelineEvent event = new TimelineEvent();
+ event.setId("start_event");
+ event.setTimestamp(ts);
+ entity.addEvent(event);
te.addEntity(entity);
TimelineEntity entity1 = new TimelineEntity();
String id1 = "hello1";
entity1.setId(id1);
entity1.setType(type);
- entity1.setCreatedTime(cTime);
+ entity1.setCreatedTime(cTime + 20L);
// add the info map in Timeline Entity
Map<String, Object> infoMap1 = new HashMap<String, Object>();
@@ -336,24 +362,34 @@ public class TestHBaseTimelineStorage {
infoMap1.put("infoMapKey2", 10);
entity1.addInfo(infoMap1);
+ // add event.
+ TimelineEvent event11 = new TimelineEvent();
+ event11.setId("end_event");
+ event11.setTimestamp(ts);
+ entity1.addEvent(event11);
+ TimelineEvent event12 = new TimelineEvent();
+ event12.setId("update_event");
+ event12.setTimestamp(ts - 10);
+ entity1.addEvent(event12);
+
+
// add the isRelatedToEntity info
- String key1 = "task";
- String value1 = "is_related_to_entity_id_here";
Set<String> isRelatedToSet1 = new HashSet<String>();
- isRelatedToSet1.add(value1);
+ isRelatedToSet1.add("relatedto3");
+ isRelatedToSet1.add("relatedto5");
Map<String, Set<String>> isRelatedTo1 = new HashMap<String, Set<String>>();
- isRelatedTo1.put(key, isRelatedToSet1);
+ isRelatedTo1.put("task1", isRelatedToSet1);
+ Set<String> isRelatedToSet11 = new HashSet<String>();
+ isRelatedToSet11.add("relatedto4");
+ isRelatedTo1.put("task2", isRelatedToSet11);
entity1.setIsRelatedToEntities(isRelatedTo1);
// add the relatesTo info
- key1 = "container";
- value1 = "relates_to_entity_id_here";
Set<String> relatesToSet1 = new HashSet<String>();
- relatesToSet1.add(value1);
- value1 = "relates_to_entity_id_here_Second";
- relatesToSet1.add(value1);
+ relatesToSet1.add("relatesto1");
+ relatesToSet1.add("relatesto2");
Map<String, Set<String>> relatesTo1 = new HashMap<String, Set<String>>();
- relatesTo1.put(key1, relatesToSet1);
+ relatesTo1.put("container", relatesToSet1);
entity1.setRelatesToEntities(relatesTo1);
// add some config entries
@@ -384,7 +420,21 @@ public class TestHBaseTimelineStorage {
String id2 = "hello2";
entity2.setId(id2);
entity2.setType(type);
- entity2.setCreatedTime(cTime);
+ entity2.setCreatedTime(cTime + 40L);
+ TimelineEvent event21 = new TimelineEvent();
+ event21.setId("update_event");
+ event21.setTimestamp(ts - 20);
+ entity2.addEvent(event21);
+ Set<String> isRelatedToSet2 = new HashSet<String>();
+ isRelatedToSet2.add("relatedto3");
+ Map<String, Set<String>> isRelatedTo2 = new HashMap<String, Set<String>>();
+ isRelatedTo2.put("task1", isRelatedToSet2);
+ entity2.setIsRelatedToEntities(isRelatedTo2);
+ Map<String, Set<String>> relatesTo3 = new HashMap<String, Set<String>>();
+ Set<String> relatesToSet14 = new HashSet<String>();
+ relatesToSet14.add("relatesto7");
+ relatesTo3.put("container2", relatesToSet14);
+ entity2.setRelatesToEntities(relatesTo3);
te.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
@@ -1113,19 +1163,585 @@ public class TestHBaseTimelineStorage {
@Test
public void testReadEntities() throws Exception {
- TimelineEntity e1 = reader.getEntity(
+ TimelineEntity entity = reader.getEntity(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111","world", "hello"),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
- assertNotNull(e1);
- assertEquals(3, e1.getConfigs().size());
- assertEquals(1, e1.getIsRelatedToEntities().size());
- Set<TimelineEntity> es1 = reader.getEntities(
+ assertNotNull(entity);
+ assertEquals(3, entity.getConfigs().size());
+ assertEquals(1, entity.getIsRelatedToEntities().size());
+ Set<TimelineEntity> entities = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111","world",
null), new TimelineEntityFilters(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
- assertEquals(3, es1.size());
+ assertEquals(3, entities.size());
+ int cfgCnt = 0;
+ int metricCnt = 0;
+ int infoCnt = 0;
+ int eventCnt = 0;
+ int relatesToCnt = 0;
+ int isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ cfgCnt += (timelineEntity.getConfigs() == null) ? 0 :
+ timelineEntity.getConfigs().size();
+ metricCnt += (timelineEntity.getMetrics() == null) ? 0 :
+ timelineEntity.getMetrics().size();
+ infoCnt += (timelineEntity.getInfo() == null) ? 0 :
+ timelineEntity.getInfo().size();
+ eventCnt += (timelineEntity.getEvents() == null) ? 0 :
+ timelineEntity.getEvents().size();
+ relatesToCnt += (timelineEntity.getRelatesToEntities() == null) ? 0 :
+ timelineEntity.getRelatesToEntities().size();
+ isRelatedToCnt += (timelineEntity.getIsRelatedToEntities() == null) ? 0 :
+ timelineEntity.getIsRelatedToEntities().size();
+ }
+ assertEquals(5, cfgCnt);
+ assertEquals(3, metricCnt);
+ assertEquals(5, infoCnt);
+ assertEquals(4, eventCnt);
+ assertEquals(4, relatesToCnt);
+ assertEquals(4, isRelatedToCnt);
+ }
+
+ @Test
+ public void testFilterEntitiesByCreatedTime() throws Exception {
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null,
+ null, null, null, null, null), new TimelineDataToRetrieve());
+ assertEquals(3, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("hello") && !entity.getId().equals("hello1") &&
+ !entity.getId().equals("hello2")) {
+ Assert.fail("Entities with ids' hello, hello1 and hello2 should be" +
+ " present");
+ }
+ }
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, 1425016502015L, null, null, null, null,
+ null, null, null), new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("hello1") &&
+ !entity.getId().equals("hello2")) {
+ Assert.fail("Entities with ids' hello1 and hello2 should be present");
+ }
+ }
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, 1425016502015L, null, null, null,
+ null, null, null), new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("hello")) {
+ Assert.fail("Entity with id hello should be present");
+ }
+ }
+ }
+
+ @Test
+ public void testReadEntitiesRelationsAndEventFiltersDefaultView()
+ throws Exception {
+ TimelineFilterList eventFilter = new TimelineFilterList();
+ eventFilter.addFilter(new TimelineExistsFilter(TimelineCompareOp.NOT_EQUAL,
+ "end_event"));
+ TimelineFilterList relatesTo = new TimelineFilterList(Operator.OR);
+ relatesTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ relatesTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList isRelatedTo = new TimelineFilterList();
+ isRelatedTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3"))));
+ isRelatedTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto5"))));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, relatesTo, isRelatedTo,
+ null, null, null, eventFilter), new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ int eventCnt = 0;
+ int isRelatedToCnt = 0;
+ int relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity id should have been hello2");
+ }
+ }
+ assertEquals(0, eventCnt);
+ assertEquals(0, isRelatedToCnt);
+ assertEquals(0, relatesToCnt);
+ }
+
+ @Test
+ public void testReadEntitiesEventFilters() throws Exception {
+ TimelineFilterList ef = new TimelineFilterList();
+ ef.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "update_event"));
+ ef.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "end_event"));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, ef),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(1, entities.size());
+ int eventCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ if (!timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity id should have been hello2");
+ }
+ }
+ assertEquals(1, eventCnt);
+
+ TimelineFilterList ef1 = new TimelineFilterList();
+ ef1.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "update_event"));
+ ef1.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "end_event"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, ef1),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ eventCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ if (!timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity id should have been hello2");
+ }
+ }
+ assertEquals(0, eventCnt);
+
+ TimelineFilterList ef2 = new TimelineFilterList();
+ ef2.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "end_event"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, ef2),
+ new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ eventCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ if (!timelineEntity.getId().equals("hello") &&
+ !timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity ids' should have been hello and hello2");
+ }
+ }
+ assertEquals(0, eventCnt);
+
+ TimelineFilterList ef3 = new TimelineFilterList();
+ ef3.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "update_event"));
+ ef3.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "dummy_event"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, ef3),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "update_event"));
+ list1.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "dummy_event"));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.EQUAL, "start_event"));
+ TimelineFilterList ef4 = new TimelineFilterList(Operator.OR, list1, list2);
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, ef4),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ eventCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ if (!timelineEntity.getId().equals("hello")) {
+ Assert.fail("Entity id should have been hello");
+ }
+ }
+ assertEquals(0, eventCnt);
+
+ TimelineFilterList ef5 = new TimelineFilterList();
+ ef5.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "update_event"));
+ ef5.addFilter(new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "end_event"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, ef5),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ eventCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ if (!timelineEntity.getId().equals("hello")) {
+ Assert.fail("Entity id should have been hello");
+ }
+ }
+ assertEquals(0, eventCnt);
+ }
+
+ @Test
+ public void testReadEntitiesIsRelatedTo() throws Exception {
+ TimelineFilterList irt = new TimelineFilterList(Operator.OR);
+ irt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task",
+ new HashSet<Object>(Arrays.asList("relatedto1"))));
+ irt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task2",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, irt, null, null, null,
+ null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ int isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("hello") &&
+ !timelineEntity.getId().equals("hello1")) {
+ Assert.fail("Entity ids' should have been hello and hello1");
+ }
+ }
+ assertEquals(3, isRelatedToCnt);
+
+ TimelineFilterList irt1 = new TimelineFilterList();
+ irt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3"))));
+ irt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, null, irt1, null, null,
+ null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity id should have been hello2");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+
+ TimelineFilterList irt2 = new TimelineFilterList(Operator.OR);
+ irt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task",
+ new HashSet<Object>(Arrays.asList("relatedto1"))));
+ irt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task2",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, null, irt2, null, null,
+ null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("hello") &&
+ !timelineEntity.getId().equals("hello1")) {
+ Assert.fail("Entity ids' should have been hello and hello1");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+
+ TimelineFilterList irt3 = new TimelineFilterList();
+ irt3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3", "relatedto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, null, irt3, null, null,
+ null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("hello1")) {
+ Assert.fail("Entity id should have been hello1");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+
+ TimelineFilterList irt4 = new TimelineFilterList();
+ irt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3"))));
+ irt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_task",
+ new HashSet<Object>(Arrays.asList("relatedto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, null, irt4, null, null,
+ null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList irt5 = new TimelineFilterList();
+ irt5.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3", "relatedto7"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, null, irt5, null, null,
+ null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task",
+ new HashSet<Object>(Arrays.asList("relatedto1"))));
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_task",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task2",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ TimelineFilterList irt6 = new TimelineFilterList(Operator.OR, list1, list2);
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, null, irt6, null, null,
+ null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("hello1")) {
+ Assert.fail("Entity id should have been hello1");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+ }
+
+ @Test
+ public void testReadEntitiesRelatesTo() throws Exception {
+ TimelineFilterList rt = new TimelineFilterList(Operator.OR);
+ rt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ rt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, rt, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ int relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello") &&
+ !timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity ids' should have been hello and hello2");
+ }
+ }
+ assertEquals(3, relatesToCnt);
+
+ TimelineFilterList rt1 = new TimelineFilterList();
+ rt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ rt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto3"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt1, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello1")) {
+ Assert.fail("Entity id should have been hello1");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList rt2 = new TimelineFilterList(Operator.OR);
+ rt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ rt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt2, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello") &&
+ !timelineEntity.getId().equals("hello2")) {
+ Assert.fail("Entity ids' should have been hello and hello2");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList rt3 = new TimelineFilterList();
+ rt3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1", "relatesto3"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt3, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello")) {
+ Assert.fail("Entity id should have been hello");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList rt4 = new TimelineFilterList();
+ rt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ rt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_container",
+ new HashSet<Object>(Arrays.asList("relatesto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt4, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList rt5 = new TimelineFilterList();
+ rt5.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatedto1", "relatesto8"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt5, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_container",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList rt6 = new TimelineFilterList(Operator.OR, list1, list2);
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt6, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello")) {
+ Assert.fail("Entity id should have been hello");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList list3 = new TimelineFilterList();
+ list3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ list3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList list4 = new TimelineFilterList();
+ list4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ list4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto2"))));
+ TimelineFilterList combinedList =
+ new TimelineFilterList(Operator.OR, list3, list4);
+ TimelineFilterList rt7 = new TimelineFilterList(Operator.AND, combinedList,
+ new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto3"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111", "world", null),
+ new TimelineEntityFilters(null, null, null, rt7, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("hello1")) {
+ Assert.fail("Entity id should have been hello1");
+ }
+ }
+ assertEquals(0, relatesToCnt);
}
@Test
@@ -1177,7 +1793,7 @@ public class TestHBaseTimelineStorage {
infoCnt += entity.getInfo().size();
}
assertEquals(0, infoCnt);
- assertEquals(2, isRelatedToCnt);
+ assertEquals(4, isRelatedToCnt);
assertEquals(3, metricsCnt);
}
@@ -1200,13 +1816,122 @@ public class TestHBaseTimelineStorage {
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
+ for (String confKey : entity.getConfigs().keySet()) {
+ assertTrue("Config key returned should start with cfg_",
+ confKey.startsWith("cfg_"));
+ }
+ }
+ assertEquals(3, cfgCnt);
+ }
+
+ @Test
+ public void testReadEntitiesConfigFilters() throws Exception {
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param2", "value2"));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value3"));
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_param2", "value2"));
+ TimelineFilterList confFilterList =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(2, entities.size());
+ int cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ }
+ assertEquals(5, cfgCnt);
+
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ }
+ assertEquals(5, cfgCnt);
+
+ TimelineFilterList confFilterList1 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList1, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(1, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
}
assertEquals(3, cfgCnt);
+
+ TimelineFilterList confFilterList2 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"),
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "config_param2", "value2"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList2, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList confFilterList3 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "dummy_config", "value1"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList3, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList confFilterList4 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList4, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList confFilterList5 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "dummy_config", "value1", false));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList5, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(3, entities.size());
}
@Test
public void testReadEntitiesConfigFilterPrefix() throws Exception {
- Map<String, String> confFilters = ImmutableMap.of("cfg_param1","value1");
+ TimelineFilterList confFilterList = new TimelineFilterList();
+ confFilterList.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
@@ -1214,12 +1939,47 @@ public class TestHBaseTimelineStorage {
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111","world", null),
new TimelineEntityFilters(null, null, null, null, null, null,
- confFilters, null, null),
+ confFilterList, null, null),
new TimelineDataToRetrieve(list, null, null));
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
cfgCnt += entity.getConfigs().size();
+ for (String confKey : entity.getConfigs().keySet()) {
+ assertTrue("Config key returned should start with cfg_",
+ confKey.startsWith("cfg_"));
+ }
+ }
+ assertEquals(2, cfgCnt);
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param2", "value2"));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value3"));
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_param2", "value2"));
+ TimelineFilterList confFilterList1 =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ TimelineFilterList confsToRetrieve =
+ new TimelineFilterList(Operator.OR,
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "config_"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList1, null, null),
+ new TimelineDataToRetrieve(confsToRetrieve, null, null));
+ assertEquals(2, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ for (String confKey : entity.getConfigs().keySet()) {
+ assertTrue("Config key returned should start with config_",
+ confKey.startsWith("config_"));
+ }
}
assertEquals(2, cfgCnt);
}
@@ -1243,47 +2003,356 @@ public class TestHBaseTimelineStorage {
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue("Metric Id returned should start with MAP1_",
+ metric.getId().startsWith("MAP1_"));
+ }
}
assertEquals(2, metricCnt);
}
@Test
- public void testReadEntitiesMetricFilterPrefix() throws Exception {
- Set<String> metricFilters = ImmutableSet.of("MAP1_SLOT_MILLIS");
- TimelineFilterList list =
- new TimelineFilterList(Operator.OR,
- new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
+ public void testReadEntitiesMetricFilters() throws Exception {
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 50000000900L));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 80000000000L));
+ list2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.EQUAL, "MAP1_BYTES", 50));
+ TimelineFilterList metricFilterList =
+ new TimelineFilterList(Operator.OR, list1, list2);
Set<TimelineEntity> entities = reader.getEntities(
- new TimelineReaderContext("cluster1","user1", "some_flow_name",
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1231111111_1111","world", null),
new TimelineEntityFilters(null, null, null, null, null, null, null,
- metricFilters, null),
- new TimelineDataToRetrieve(null, list, null));
- assertEquals(1, entities.size());
+ metricFilterList, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(2, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
metricCnt += entity.getMetrics().size();
}
- assertEquals(1, metricCnt);
- }
+ assertEquals(3, metricCnt);
- @Test
- public void testReadApps() throws Exception {
- TimelineEntity e1 = reader.getEntity(
+ entities = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
- 1002345678919L, "application_1111111111_2222",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ }
+ assertEquals(3, metricCnt);
+
+ TimelineFilterList metricFilterList1 = new TimelineFilterList(
+ new TimelineCompareFilter(
+ TimelineCompareOp.LESS_OR_EQUAL, "MAP_SLOT_MILLIS", 80000000000L),
+ new TimelineCompareFilter(
+ TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList1, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(1, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ }
+ assertEquals(2, metricCnt);
+
+ TimelineFilterList metricFilterList2 = new TimelineFilterList(
+ new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 40000000000L),
+ new TimelineCompareFilter(
+ TimelineCompareOp.NOT_EQUAL, "MAP1_BYTES", 30));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList2, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList metricFilterList3 = new TimelineFilterList(
+ new TimelineCompareFilter(
+ TimelineCompareOp.EQUAL, "dummy_metric", 5));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList3, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList metricFilterList4 = new TimelineFilterList(
+ new TimelineCompareFilter(
+ TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList4, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList metricFilterList5 = new TimelineFilterList(
+ new TimelineCompareFilter(
+ TimelineCompareOp.NOT_EQUAL, "dummy_metric", 5, false));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList5, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(3, entities.size());
+ }
+
+ @Test
+ public void testReadEntitiesMetricFilterPrefix() throws Exception {
+ TimelineFilterList metricFilterList = new TimelineFilterList();
+ metricFilterList.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 0L));
+ TimelineFilterList list =
+ new TimelineFilterList(Operator.OR,
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList, null),
+ new TimelineDataToRetrieve(null, list, null));
+ assertEquals(1, entities.size());
+ int metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue("Metric Id returned should start with MAP1_",
+ metric.getId().startsWith("MAP1_"));
+ }
+ }
+ assertEquals(1, metricCnt);
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, "MAP1_SLOT_MILLIS", 50000000900L));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, "MAP_SLOT_MILLIS", 80000000000L));
+ list2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.EQUAL, "MAP1_BYTES", 50));
+ TimelineFilterList metricFilterList1 =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR,
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList1, null),
+ new TimelineDataToRetrieve(
+ null, metricsToRetrieve, EnumSet.of(Field.METRICS)));
+ assertEquals(2, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue("Metric Id returned should start with MAP1_",
+ metric.getId().startsWith("MAP1_"));
+ }
+ }
+ assertEquals(2, metricCnt);
+ }
+
+ @Test
+ public void testReadEntitiesInfoFilters() throws Exception {
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "infoMapKey3", 71.4));
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "infoMapKey1", "infoMapValue2"));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "infoMapKey1", "infoMapValue1"));
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "infoMapKey2", 10));
+ TimelineFilterList infoFilterList =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList,
+ null, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO)));
+ assertEquals(2, entities.size());
+ int infoCnt = 0;
+ for (TimelineEntity entity : entities) {
+ infoCnt += entity.getInfo().size();
+ }
+ assertEquals(5, infoCnt);
+
+ TimelineFilterList infoFilterList1 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "infoMapKey1", "infoMapValue1"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList1,
+ null, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO)));
+ assertEquals(1, entities.size());
+ infoCnt = 0;
+ for (TimelineEntity entity : entities) {
+ infoCnt += entity.getInfo().size();
+ }
+ assertEquals(3, infoCnt);
+
+ TimelineFilterList infoFilterList2 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "infoMapKey1", "infoMapValue2"),
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "infoMapKey3", 71.4));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList2,
+ null, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList infoFilterList3 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "dummy_info", "some_value"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList3,
+ null, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList infoFilterList4 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "dummy_info", "some_value"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList4,
+ null, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList infoFilterList5 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "dummy_info", "some_value", false));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList5,
+ null, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.INFO)));
+ assertEquals(3, entities.size());
+ }
+
+ @Test
+ public void testReadApps() throws Exception {
+ TimelineEntity entity = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1111111111_2222",
TimelineEntityType.YARN_APPLICATION.toString(), null),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
- assertNotNull(e1);
- assertEquals(3, e1.getConfigs().size());
- assertEquals(1, e1.getIsRelatedToEntities().size());
- Set<TimelineEntity> es1 = reader.getEntities(
+ assertNotNull(entity);
+ assertEquals(3, entity.getConfigs().size());
+ assertEquals(1, entity.getIsRelatedToEntities().size());
+ Set<TimelineEntity> entities = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
null),
new TimelineEntityFilters(),
new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
- assertEquals(3, es1.size());
+ assertEquals(3, entities.size());
+ int cfgCnt = 0;
+ int metricCnt = 0;
+ int infoCnt = 0;
+ int eventCnt = 0;
+ int relatesToCnt = 0;
+ int isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ cfgCnt += (timelineEntity.getConfigs() == null) ? 0 :
+ timelineEntity.getConfigs().size();
+ metricCnt += (timelineEntity.getMetrics() == null) ? 0 :
+ timelineEntity.getMetrics().size();
+ infoCnt += (timelineEntity.getInfo() == null) ? 0 :
+ timelineEntity.getInfo().size();
+ eventCnt += (timelineEntity.getEvents() == null) ? 0 :
+ timelineEntity.getEvents().size();
+ relatesToCnt += (timelineEntity.getRelatesToEntities() == null) ? 0 :
+ timelineEntity.getRelatesToEntities().size();
+ isRelatedToCnt += (timelineEntity.getIsRelatedToEntities() == null) ? 0 :
+ timelineEntity.getIsRelatedToEntities().size();
+ }
+ assertEquals(5, cfgCnt);
+ assertEquals(3, metricCnt);
+ assertEquals(5, infoCnt);
+ assertEquals(4, eventCnt);
+ assertEquals(4, relatesToCnt);
+ assertEquals(4, isRelatedToCnt);
+ }
+
+ @Test
+ public void testFilterAppsByCreatedTime() throws Exception {
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, 1425016502000L, 1425016502040L, null,
+ null, null, null, null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(3, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("application_1111111111_2222") &&
+ !entity.getId().equals("application_1111111111_3333") &&
+ !entity.getId().equals("application_1111111111_4444")) {
+ Assert.fail("Entities with ids' application_1111111111_2222, " +
+ "application_1111111111_3333 and application_1111111111_4444" +
+ " should be present");
+ }
+ }
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, 1425016502015L, null, null, null, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("application_1111111111_3333") &&
+ !entity.getId().equals("application_1111111111_4444")) {
+ Assert.fail("Apps with ids' application_1111111111_3333 and" +
+ " application_1111111111_4444 should be present");
+ }
+ }
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, 1425016502015L, null, null, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("application_1111111111_2222")) {
+ Assert.fail("App with id application_1111111111_2222 should" +
+ " be present");
+ }
+ }
}
@Test
@@ -1339,11 +2408,649 @@ public class TestHBaseTimelineStorage {
infoCnt += entity.getInfo().size();
}
assertEquals(0, infoCnt);
- assertEquals(2, isRelatedToCnt);
+ assertEquals(4, isRelatedToCnt);
assertEquals(3, metricsCnt);
}
@Test
+ public void testReadAppsIsRelatedTo() throws Exception {
+ TimelineFilterList irt = new TimelineFilterList(Operator.OR);
+ irt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task",
+ new HashSet<Object>(Arrays.asList("relatedto1"))));
+ irt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task2",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt, null, null, null,
+ null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ int isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_2222") &&
+ !timelineEntity.getId().equals("application_1111111111_3333")) {
+ Assert.fail("Entity ids' should have been application_1111111111_2222"
+ + " and application_1111111111_3333");
+ }
+ }
+ assertEquals(3, isRelatedToCnt);
+
+ TimelineFilterList irt1 = new TimelineFilterList();
+ irt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3"))));
+ irt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt1, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_4444")) {
+ Assert.fail("Entity id should have been application_1111111111_4444");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+
+ TimelineFilterList irt2 = new TimelineFilterList(Operator.OR);
+ irt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task",
+ new HashSet<Object>(Arrays.asList("relatedto1"))));
+ irt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task2",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt2, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_2222") &&
+ !timelineEntity.getId().equals("application_1111111111_3333")) {
+ Assert.fail("Entity ids' should have been application_1111111111_2222"
+ + " and application_1111111111_3333");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+
+ TimelineFilterList irt3 = new TimelineFilterList();
+ irt3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3", "relatedto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt3, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_3333")) {
+ Assert.fail("Entity id should have been application_1111111111_3333");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+
+ TimelineFilterList irt4 = new TimelineFilterList();
+ irt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3"))));
+ irt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_task",
+ new HashSet<Object>(Arrays.asList("relatedto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt4, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList irt5 = new TimelineFilterList();
+ irt5.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3", "relatedto7"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt5, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task",
+ new HashSet<Object>(Arrays.asList("relatedto1"))));
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_task",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task2",
+ new HashSet<Object>(Arrays.asList("relatedto4"))));
+ TimelineFilterList irt6 = new TimelineFilterList(Operator.OR, list1, list2);
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, irt6, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ isRelatedToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_3333")) {
+ Assert.fail("Entity id should have been application_1111111111_3333");
+ }
+ }
+ assertEquals(0, isRelatedToCnt);
+ }
+
+
+ @Test
+ public void testReadAppsRelatesTo() throws Exception {
+ TimelineFilterList rt = new TimelineFilterList(Operator.OR);
+ rt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ rt.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ int relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_2222") &&
+ !timelineEntity.getId().equals("application_1111111111_4444")) {
+ Assert.fail("Entity ids' should have been application_1111111111_2222"
+ + " and application_1111111111_4444");
+ }
+ }
+ assertEquals(3, relatesToCnt);
+
+ TimelineFilterList rt1 = new TimelineFilterList();
+ rt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ rt1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto3"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt1, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_3333")) {
+ Assert.fail("Entity id should have been application_1111111111_3333");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList rt2 = new TimelineFilterList(Operator.OR);
+ rt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ rt2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt2, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_2222") &&
+ !timelineEntity.getId().equals("application_1111111111_4444")) {
+ Assert.fail("Entity ids' should have been application_1111111111_2222"
+ + " and application_1111111111_4444");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList rt3 = new TimelineFilterList();
+ rt3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1", "relatesto3"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt3, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_2222")) {
+ Assert.fail("Entity id should have been application_1111111111_2222");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList rt4 = new TimelineFilterList();
+ rt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ rt4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_container",
+ new HashSet<Object>(Arrays.asList("relatesto5"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt4, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList rt5 = new TimelineFilterList();
+ rt5.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatedto1", "relatesto8"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt5, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(0, entities.size());
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ list1.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "dummy_container",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList rt6 = new TimelineFilterList(Operator.OR, list1, list2);
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt6, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_2222")) {
+ Assert.fail("Entity id should have been application_1111111111_2222");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+
+ TimelineFilterList list3 = new TimelineFilterList();
+ list3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ list3.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList list4 = new TimelineFilterList();
+ list4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto1"))));
+ list4.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto2"))));
+ TimelineFilterList combinedList =
+ new TimelineFilterList(Operator.OR, list3, list4);
+ TimelineFilterList rt7 = new TimelineFilterList(Operator.AND, combinedList,
+ new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "container",
+ new HashSet<Object>(Arrays.asList("relatesto3"))));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, rt7, null, null, null, null,
+ null),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_3333")) {
+ Assert.fail("Entity id should have been application_1111111111_3333");
+ }
+ }
+ assertEquals(0, relatesToCnt);
+ }
+
+ @Test
+ public void testReadAppsRelationsAndEventFiltersDefaultView()
+ throws Exception {
+ TimelineFilterList eventFilter = new TimelineFilterList();
+ eventFilter.addFilter(new TimelineExistsFilter(TimelineCompareOp.NOT_EQUAL,
+ "end_event"));
+ TimelineFilterList relatesTo = new TimelineFilterList(Operator.OR);
+ relatesTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container2",
+ new HashSet<Object>(Arrays.asList("relatesto7"))));
+ relatesTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "container1",
+ new HashSet<Object>(Arrays.asList("relatesto4"))));
+ TimelineFilterList isRelatedTo = new TimelineFilterList();
+ isRelatedTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto3"))));
+ isRelatedTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.NOT_EQUAL, "task1",
+ new HashSet<Object>(Arrays.asList("relatedto5"))));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, relatesTo, isRelatedTo,
+ null, null, null, eventFilter),
+ new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ int eventCnt = 0;
+ int isRelatedToCnt = 0;
+ int relatesToCnt = 0;
+ for (TimelineEntity timelineEntity : entities) {
+ eventCnt += timelineEntity.getEvents().size();
+ isRelatedToCnt += timelineEntity.getIsRelatedToEntities().size();
+ relatesToCnt += timelineEntity.getRelatesToEntities().size();
+ if (!timelineEntity.getId().equals("application_1111111111_4444")) {
+ Assert.fail("Entity id should have been application_1111111111_4444");
+ }
+ }
+ assertEquals(0, eventCnt);
+ assertEquals(0, isRelatedToCnt);
+ assertEquals(0, relatesToCnt);
+ }
+
+ @Test
+ public void testReadAppsConfigFilters() throws Exception {
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value1"));
+ list1.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param2", "value2"));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "cfg_param1", "value3"));
+ list2.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_param2", "value2"));
+ TimelineFilterList confFilterList =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(2, entities.size());
+ int cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ }
+ assertEquals(5, cfgCnt);
+
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ }
+ assertEquals(5, cfgCnt);
+
+ TimelineFilterList confFilterList1 = new TimelineFilterList(
+ new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "cfg_param1", "value1"));
+ entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList1, null, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.CONFIGS)));
+ assertEquals(1, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntit
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and
checkstyle issues in timelineservice code (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
index 50136de..663a18a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineWriter.java
@@ -42,12 +42,13 @@ public interface TimelineWriter extends Service {
* @param userId context user ID
* @param flowName context flow name
* @param flowVersion context flow version
- * @param flowRunId
- * @param appId context app ID
+ * @param flowRunId run id for the flow.
+ * @param appId context app ID.
* @param data
* a {@link TimelineEntities} object.
* @return a {@link TimelineWriteResponse} object.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while storing
+ * or writing entities to the backend storage.
*/
TimelineWriteResponse write(String clusterId, String userId,
String flowName, String flowVersion, long flowRunId, String appId,
@@ -65,8 +66,11 @@ public interface TimelineWriter extends Service {
* a {@link TimelineEntity} object
* a {@link TimelineAggregationTrack} enum
* value.
+ * @param track Specifies the track or dimension along which aggregation would
+ * occur. Includes USER, FLOW, QUEUE, etc.
* @return a {@link TimelineWriteResponse} object.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while aggregating
+ * entities to the backend storage.
*/
TimelineWriteResponse aggregate(TimelineEntity data,
TimelineAggregationTrack track) throws IOException;
@@ -76,7 +80,8 @@ public interface TimelineWriter extends Service {
* written to the storage when the method returns. This may be a potentially
* time-consuming operation, and should be used judiciously.
*
- * @throws IOException
+ * @throws IOException if there is any exception encountered while flushing
+ * entities to the backend storage.
*/
void flush() throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
index c03c9b6..5734389 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumn.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
public enum ApplicationColumn implements Column<ApplicationTable> {
/**
- * App id
+ * App id.
*/
ID(ApplicationColumnFamily.INFO, "id"),
@@ -84,7 +84,7 @@ public enum ApplicationColumn implements Column<ApplicationTable> {
/**
* Retrieve an {@link ApplicationColumn} given a name, or null if there is no
* match. The following holds true: {@code columnFor(x) == columnFor(y)} if
- * and only if {@code x.equals(y)} or {@code (x == y == null)}
+ * and only if {@code x.equals(y)} or {@code (x == y == null)}.
*
* @param columnQualifier Name of the column to retrieve
* @return the corresponding {@link ApplicationColumn} or null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
index 056e51f..9120f3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationColumnPrefix.java
@@ -54,7 +54,7 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
INFO(ApplicationColumnFamily.INFO, "i"),
/**
- * Lifecycle events for an application
+ * Lifecycle events for an application.
*/
EVENT(ApplicationColumnFamily.INFO, "e"),
@@ -214,7 +214,7 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
* is to facilitate returning byte arrays of values that were not
* Strings. If they can be treated as Strings, you should use
* {@link #readResults(Result)} instead.
- * @throws IOException
+ * @throws IOException if any problem occurs while reading results.
*/
public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
throws IOException {
@@ -276,8 +276,8 @@ public enum ApplicationColumnPrefix implements ColumnPrefix<ApplicationTable> {
for (ApplicationColumnPrefix acp : ApplicationColumnPrefix.values()) {
// Find a match based column family and on name.
if (acp.columnFamily.equals(columnFamily)
- && (((columnPrefix == null) && (acp.getColumnPrefix() == null)) || (acp
- .getColumnPrefix().equals(columnPrefix)))) {
+ && (((columnPrefix == null) && (acp.getColumnPrefix() == null)) ||
+ (acp.getColumnPrefix().equals(columnPrefix)))) {
return acp;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
index 3c182b9..ad2aa7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationRowKey.java
@@ -62,11 +62,11 @@ public class ApplicationRowKey {
/**
* Constructs a row key prefix for the application table as follows:
- * {@code clusterId!userName!flowName!}
+ * {@code clusterId!userName!flowName!}.
*
- * @param clusterId
- * @param userId
- * @param flowName
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
@@ -78,12 +78,12 @@ public class ApplicationRowKey {
/**
* Constructs a row key prefix for the application table as follows:
- * {@code clusterId!userName!flowName!flowRunId!}
+ * {@code clusterId!userName!flowName!flowRunId!}.
*
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
+ * @param flowRunId Run Id for the flow.
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
@@ -96,13 +96,13 @@ public class ApplicationRowKey {
/**
* Constructs a row key for the application table as follows:
- * {@code clusterId!userName!flowName!flowRunId!AppId}
+ * {@code clusterId!userName!flowName!flowRunId!AppId}.
*
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
- * @param appId
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
+ * @param flowRunId Run Id for the flow.
+ * @param appId App Id.
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String userId,
@@ -119,6 +119,9 @@ public class ApplicationRowKey {
/**
* Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey Byte representation of row key.
+ * @return An <cite>ApplicationRowKey</cite> object.
*/
public static ApplicationRowKey parseRowKey(byte[] rowKey) {
byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
index 681c200..a02f768 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/ApplicationTable.java
@@ -68,28 +68,28 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* </pre>
*/
public class ApplicationTable extends BaseTable<ApplicationTable> {
- /** application prefix */
+ /** application prefix. */
private static final String PREFIX =
YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".application";
- /** config param name that specifies the application table name */
+ /** config param name that specifies the application table name. */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
/**
* config param name that specifies the TTL for metrics column family in
- * application table
+ * application table.
*/
private static final String METRICS_TTL_CONF_NAME = PREFIX
+ ".table.metrics.ttl";
- /** default value for application table name */
+ /** default value for application table name. */
private static final String DEFAULT_TABLE_NAME =
"timelineservice.application";
- /** default TTL is 30 days for metrics timeseries */
+ /** default TTL is 30 days for metrics timeseries. */
private static final int DEFAULT_METRICS_TTL = 2592000;
- /** default max number of versions */
+ /** default max number of versions. */
private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
private static final Log LOG = LogFactory.getLog(ApplicationTable.class);
@@ -139,8 +139,8 @@ public class ApplicationTable extends BaseTable<ApplicationTable> {
metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
DEFAULT_METRICS_TTL));
- applicationTableDescp
- .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ applicationTableDescp.setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
applicationTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
admin.createTable(applicationTableDescp,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
index c60e6f5..eda14e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/application/package-info.java
@@ -16,6 +16,10 @@
* limitations under the License.
*/
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.application
+ * contains classes related to implementation for application table.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.yarn.server.timelineservice.storage.application;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
index 7f1ecaf..a8e1c66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumn.java
@@ -35,17 +35,17 @@ import java.io.IOException;
public enum AppToFlowColumn implements Column<AppToFlowTable> {
/**
- * The flow ID
+ * The flow ID.
*/
FLOW_ID(AppToFlowColumnFamily.MAPPING, "flow_id"),
/**
- * The flow run ID
+ * The flow run ID.
*/
FLOW_RUN_ID(AppToFlowColumnFamily.MAPPING, "flow_run_id"),
/**
- * The user
+ * The user.
*/
USER_ID(AppToFlowColumnFamily.MAPPING, "user_id");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
index d722d15..f3f045e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowColumnFamily.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
*/
public enum AppToFlowColumnFamily implements ColumnFamily<AppToFlowTable> {
/**
- * Mapping column family houses known columns such as flowName and flowRunId
+ * Mapping column family houses known columns such as flowName and flowRunId.
*/
MAPPING("m");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
index 133952e..3085bb1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowRowKey.java
@@ -43,10 +43,10 @@ public class AppToFlowRowKey {
/**
* Constructs a row key prefix for the app_flow table as follows:
- * {@code clusterId!AppId}
+ * {@code clusterId!AppId}.
*
- * @param clusterId
- * @param appId
+ * @param clusterId Cluster Id.
+ * @param appId Application Id.
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String appId) {
@@ -57,6 +57,9 @@ public class AppToFlowRowKey {
/**
* Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey a rowkey represented as a byte array.
+ * @return an <cite>AppToFlowRowKey</cite> object.
*/
public static AppToFlowRowKey parseRowKey(byte[] rowKey) {
byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
index 868c6f3..301cf99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/AppToFlowTable.java
@@ -58,14 +58,14 @@ import java.io.IOException;
* </pre>
*/
public class AppToFlowTable extends BaseTable<AppToFlowTable> {
- /** app_flow prefix */
+ /** app_flow prefix. */
private static final String PREFIX =
YarnConfiguration.TIMELINE_SERVICE_PREFIX + "app-flow";
- /** config param name that specifies the app_flow table name */
+ /** config param name that specifies the app_flow table name. */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
- /** default value for app_flow table name */
+ /** default value for app_flow table name. */
private static final String DEFAULT_TABLE_NAME = "timelineservice.app_flow";
private static final Log LOG = LogFactory.getLog(AppToFlowTable.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
index df7ffc1..f01d982 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/apptoflow/package-info.java
@@ -15,6 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow
+ * contains classes related to implementation for app to flow table.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
index 9545438..8581aa4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BaseTable.java
@@ -39,7 +39,7 @@ import org.apache.hadoop.hbase.client.Table;
public abstract class BaseTable<T> {
/**
- * Name of config variable that is used to point to this table
+ * Name of config variable that is used to point to this table.
*/
private final String tableNameConfName;
@@ -52,6 +52,8 @@ public abstract class BaseTable<T> {
/**
* @param tableNameConfName name of config variable that is used to point to
* this table.
+ * @param defaultTableName Default table name if table from config is not
+ * found.
*/
protected BaseTable(String tableNameConfName, String defaultTableName) {
this.tableNameConfName = tableNameConfName;
@@ -61,10 +63,11 @@ public abstract class BaseTable<T> {
/**
* Used to create a type-safe mutator for this table.
*
- * @param hbaseConf used to read table name
+ * @param hbaseConf used to read table name.
* @param conn used to create a table from.
* @return a type safe {@link BufferedMutator} for the entity table.
- * @throws IOException
+ * @throws IOException if any exception occurs while creating mutator for the
+ * table.
*/
public TypedBufferedMutator<T> getTableMutator(Configuration hbaseConf,
Connection conn) throws IOException {
@@ -88,7 +91,7 @@ public abstract class BaseTable<T> {
* @param conn used to create table from
* @param scan that specifies what you want to read from this table.
* @return scanner for the table.
- * @throws IOException
+ * @throws IOException if any exception occurs while getting the scanner.
*/
public ResultScanner getResultScanner(Configuration hbaseConf,
Connection conn, Scan scan) throws IOException {
@@ -102,7 +105,7 @@ public abstract class BaseTable<T> {
* @param conn used to create table from
* @param get that specifies what single row you want to get from this table
* @return result of get operation
- * @throws IOException
+ * @throws IOException if any exception occurs while getting the result.
*/
public Result getResult(Configuration hbaseConf, Connection conn, Get get)
throws IOException {
@@ -113,7 +116,8 @@ public abstract class BaseTable<T> {
/**
* Get the table name for this table.
*
- * @param hbaseConf
+ * @param hbaseConf HBase configuration from which table name will be fetched.
+ * @return A {@link TableName} object.
*/
public TableName getTableName(Configuration hbaseConf) {
TableName table =
@@ -126,8 +130,9 @@ public abstract class BaseTable<T> {
* Used to create the table in HBase. Should be called only once (per HBase
* instance).
*
- * @param admin
- * @param hbaseConf
+ * @param admin Used for doing HBase table operations.
+ * @param hbaseConf Hbase configuration.
+ * @throws IOException if any exception occurs while creating the table.
*/
public abstract void createTable(Admin admin, Configuration hbaseConf)
throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
index fe8f9c6..cf469a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/BufferedMutatorDelegator.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.client.BufferedMutator;
import org.apache.hadoop.hbase.client.Mutation;
/**
- * To be used to wrap an actual {@link BufferedMutator} in a type safe manner
+ * To be used to wrap an actual {@link BufferedMutator} in a type safe manner.
*
* @param <T> The class referring to the table to be written to.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
index 64c1cda..1f0b48f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Column.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import java.io.IOException;
-import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
@@ -39,25 +38,26 @@ public interface Column<T> {
* column.
* @param timestamp version timestamp. When null the server timestamp will be
* used.
- * @param attributes Map of attributes for this mutation. used in the coprocessor
- * to set/read the cell tags. Can be null.
+ * @param attributes Map of attributes for this mutation. used in the
+ * coprocessor to set/read the cell tags. Can be null.
* @param inputValue the value to write to the rowKey and column qualifier.
* Nothing gets written when null.
- * @throws IOException
+ * @throws IOException if there is any exception encountered during store.
*/
- public void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
+ void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
Long timestamp, Object inputValue, Attribute... attributes)
throws IOException;
/**
* Get the latest version of this specified column. Note: this call clones the
- * value content of the hosting {@link Cell}.
+ * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
*
* @param result Cannot be null
* @return result object (can be cast to whatever object was written to), or
* null when result doesn't contain this column.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while reading
+ * result.
*/
- public Object readResult(Result result) throws IOException;
+ Object readResult(Result result) throws IOException;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
index c84c016..452adcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnFamily.java
@@ -29,6 +29,6 @@ public interface ColumnFamily<T> {
*
* @return a clone of the byte representation of the column family.
*/
- public byte[] getBytes();
+ byte[] getBytes();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
index 1e63ce5..15bb818 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnHelper.java
@@ -26,7 +26,6 @@ import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.util.Bytes;
@@ -82,7 +81,9 @@ public class ColumnHelper<T> {
* @param inputValue
* the value to write to the rowKey and column qualifier. Nothing
* gets written when null.
- * @throws IOException
+ * @param attributes Attributes to be set for HBase Put.
+ * @throws IOException if any problem occurs during store operation(sending
+ * mutation to table).
*/
public void store(byte[] rowKey, TypedBufferedMutator<?> tableMutator,
byte[] columnQualifier, Long timestamp, Object inputValue,
@@ -140,13 +141,13 @@ public class ColumnHelper<T> {
/**
* Get the latest version of this specified column. Note: this call clones the
- * value content of the hosting {@link Cell}.
+ * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
*
* @param result from which to read the value. Cannot be null
* @param columnQualifierBytes referring to the column to be read.
* @return latest version of the specified column of whichever object was
* written.
- * @throws IOException
+ * @throws IOException if any problem occurs while reading result.
*/
public Object readResult(Result result, byte[] columnQualifierBytes)
throws IOException {
@@ -167,9 +168,9 @@ public class ColumnHelper<T> {
* columns are returned.
* @param <V> the type of the values. The values will be cast into that type.
* @return the cell values at each respective time in for form
- * {idA={timestamp1->value1}, idA={timestamp2->value2},
- * idB={timestamp3->value3}, idC={timestamp1->value4}}
- * @throws IOException
+ * {@literal {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}}
+ * @throws IOException if any problem occurs while reading results.
*/
@SuppressWarnings("unchecked")
public <V> NavigableMap<String, NavigableMap<Long, V>>
@@ -180,8 +181,9 @@ public class ColumnHelper<T> {
new TreeMap<String, NavigableMap<Long, V>>();
if (result != null) {
- NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
- result.getMap();
+ NavigableMap<
+ byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
+ result.getMap();
NavigableMap<byte[], NavigableMap<Long, byte[]>> columnCellMap =
resultMap.get(columnFamilyBytes);
@@ -240,7 +242,7 @@ public class ColumnHelper<T> {
* back and forth from Strings, you should use
* {@link #readResultsHavingCompoundColumnQualifiers(Result, byte[])}
* instead.
- * @throws IOException
+ * @throws IOException if any problem occurs while reading results.
*/
public Map<String, Object> readResults(Result result,
byte[] columnPrefixBytes) throws IOException {
@@ -294,7 +296,7 @@ public class ColumnHelper<T> {
* non-null column prefix bytes, the column qualifier is returned as
* a list of parts, each part a byte[]. This is to facilitate
* returning byte arrays of values that were not Strings.
- * @throws IOException
+ * @throws IOException if any problem occurs while reading results.
*/
public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result,
byte[] columnPrefixBytes) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
index 0f3ac4e..f221b31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ColumnPrefix.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.Map;
import java.util.NavigableMap;
-import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
@@ -48,7 +47,8 @@ public interface ColumnPrefix<T> {
* coprocessor to set/read the cell tags.
* @param inputValue the value to write to the rowKey and column qualifier.
* Nothing gets written when null.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while doing
+ * store operation(sending mutation to the table).
*/
void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
byte[] qualifier, Long timestamp, Object inputValue,
@@ -69,7 +69,8 @@ public interface ColumnPrefix<T> {
* coprocessor to set/read the cell tags.
* @param inputValue the value to write to the rowKey and column qualifier.
* Nothing gets written when null.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while doing
+ * store operation(sending mutation to the table).
*/
void store(byte[] rowKey, TypedBufferedMutator<T> tableMutator,
String qualifier, Long timestamp, Object inputValue,
@@ -77,14 +78,15 @@ public interface ColumnPrefix<T> {
/**
* Get the latest version of this specified column. Note: this call clones the
- * value content of the hosting {@link Cell}.
+ * value content of the hosting {@link org.apache.hadoop.hbase.Cell Cell}.
*
* @param result Cannot be null
* @param qualifier column qualifier. Nothing gets read when null.
* @return result object (can be cast to whatever object was written to) or
* null when specified column qualifier for this prefix doesn't exist
* in the result.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while reading
+ * result.
*/
Object readResult(Result result, String qualifier) throws IOException;
@@ -92,7 +94,8 @@ public interface ColumnPrefix<T> {
* @param result from which to read columns
* @return the latest values of columns in the column family with this prefix
* (or all of them if the prefix value is null).
- * @throws IOException
+ * @throws IOException if there is any exception encountered while reading
+ * results.
*/
Map<String, Object> readResults(Result result) throws IOException;
@@ -100,9 +103,10 @@ public interface ColumnPrefix<T> {
* @param result from which to reads data with timestamps
* @param <V> the type of the values. The values will be cast into that type.
* @return the cell values at each respective time in for form
- * {idA={timestamp1->value1}, idA={timestamp2->value2},
- * idB={timestamp3->value3}, idC={timestamp1->value4}}
- * @throws IOException
+ * {@literal {idA={timestamp1->value1}, idA={timestamp2->value2},
+ * idB={timestamp3->value3}, idC={timestamp1->value4}}}
+ * @throws IOException if there is any exception encountered while reading
+ * result.
*/
<V> NavigableMap<String, NavigableMap<Long, V>>
readResultsWithTimestamps(Result result) throws IOException;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
index cdb8619..48c56f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/LongConverter.java
@@ -55,8 +55,9 @@ public final class LongConverter implements NumericValueConverter {
/**
* Compares two numbers as longs. If either number is null, it will be taken
* as 0.
- * @param num1
- * @param num2
+ *
+ * @param num1 the first {@code Long} to compare.
+ * @param num2 the second {@code Long} to compare.
* @return -1 if num1 is less than num2, 0 if num1 is equal to num2 and 1 if
* num1 is greater than num2.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
index 70964cd..8fb6536 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/NumericValueConverter.java
@@ -29,9 +29,10 @@ public interface NumericValueConverter extends ValueConverter,
/**
* Adds two or more numbers. If either of the numbers are null, it is taken as
* 0.
- * @param num1
- * @param num2
- * @param numbers
+ *
+ * @param num1 the first number to add.
+ * @param num2 the second number to add.
+ * @param numbers Rest of the numbers to be added.
* @return result after adding up the numbers.
*/
Number add(Number num1, Number num2, Number...numbers);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
index 16c03a3..3dc5f51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/OfflineAggregationInfo.java
@@ -33,21 +33,24 @@ import java.sql.SQLException;
*/
public final class OfflineAggregationInfo {
/**
- * Default flow level aggregation table name
+ * Default flow level aggregation table name.
*/
@VisibleForTesting
public static final String FLOW_AGGREGATION_TABLE_NAME
= "yarn_timeline_flow_aggregation";
/**
- * Default user level aggregation table name
+ * Default user level aggregation table name.
*/
public static final String USER_AGGREGATION_TABLE_NAME
= "yarn_timeline_user_aggregation";
// These lists are not taking effects in table creations.
- private static final String[] FLOW_AGGREGATION_PK_LIST =
- { "user", "cluster", "flow_name" };
- private static final String[] USER_AGGREGATION_PK_LIST = { "user", "cluster"};
+ private static final String[] FLOW_AGGREGATION_PK_LIST = {
+ "user", "cluster", "flow_name"
+ };
+ private static final String[] USER_AGGREGATION_PK_LIST = {
+ "user", "cluster"
+ };
private final String tableName;
private final String[] primaryKeyList;
@@ -81,30 +84,32 @@ public final class OfflineAggregationInfo {
public static final OfflineAggregationInfo FLOW_AGGREGATION =
new OfflineAggregationInfo(FLOW_AGGREGATION_TABLE_NAME,
- FLOW_AGGREGATION_PK_LIST, new PrimaryKeyStringSetter() {
- @Override
- public int setValues(PreparedStatement ps,
- TimelineCollectorContext context, String[] extraInfo, int startPos)
- throws SQLException {
- int idx = startPos;
- ps.setString(idx++, context.getUserId());
- ps.setString(idx++, context.getClusterId());
- ps.setString(idx++, context.getFlowName());
- return idx;
- }
- });
+ FLOW_AGGREGATION_PK_LIST,
+ new PrimaryKeyStringSetter() {
+ @Override
+ public int setValues(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo,
+ int startPos) throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++, context.getClusterId());
+ ps.setString(idx++, context.getFlowName());
+ return idx;
+ }
+ });
public static final OfflineAggregationInfo USER_AGGREGATION =
new OfflineAggregationInfo(USER_AGGREGATION_TABLE_NAME,
- USER_AGGREGATION_PK_LIST, new PrimaryKeyStringSetter() {
- @Override
- public int setValues(PreparedStatement ps,
- TimelineCollectorContext context, String[] extraInfo, int startPos)
- throws SQLException {
- int idx = startPos;
- ps.setString(idx++, context.getUserId());
- ps.setString(idx++, context.getClusterId());
- return idx;
- }
- });
+ USER_AGGREGATION_PK_LIST,
+ new PrimaryKeyStringSetter() {
+ @Override
+ public int setValues(PreparedStatement ps,
+ TimelineCollectorContext context, String[] extraInfo,
+ int startPos) throws SQLException {
+ int idx = startPos;
+ ps.setString(idx++, context.getUserId());
+ ps.setString(idx++, context.getClusterId());
+ return idx;
+ }
+ });
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
index 2cb6c08..8a2e01a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Range.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.common;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+/**
+ * Encapsulates a range with start and end indices.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class Range {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
index 1e82494..a81c717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/Separator.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.util.Bytes;
public enum Separator {
/**
- * separator in key or column qualifier fields
+ * separator in key or column qualifier fields.
*/
QUALIFIERS("!", "%0$"),
@@ -53,7 +53,7 @@ public enum Separator {
private final String value;
/**
- * The URLEncoded version of this separator
+ * The URLEncoded version of this separator.
*/
private final String encodedValue;
@@ -63,7 +63,7 @@ public enum Separator {
private final byte[] bytes;
/**
- * The value quoted so that it can be used as a safe regex
+ * The value quoted so that it can be used as a safe regex.
*/
private final String quotedValue;
@@ -99,7 +99,7 @@ public enum Separator {
/**
* Used to make token safe to be used with this separator without collisions.
*
- * @param token
+ * @param token Token to be encoded.
* @return the token with any occurrences of this separator URLEncoded.
*/
public String encode(String token) {
@@ -111,7 +111,9 @@ public enum Separator {
}
/**
- * @param token
+ * Decode the token encoded using {@link #encode}.
+ *
+ * @param token Token to be decoded.
* @return the token with any occurrences of the encoded separator replaced by
* the separator itself.
*/
@@ -193,7 +195,7 @@ public enum Separator {
* Returns a single byte array containing all of the individual arrays
* components separated by this separator.
*
- * @param components
+ * @param components Byte array components to be joined together.
* @return byte array after joining the components
*/
public byte[] join(byte[]... components) {
@@ -287,8 +289,8 @@ public enum Separator {
public Collection<String> splitEncoded(String compoundValue) {
List<String> result = new ArrayList<String>();
if (compoundValue != null) {
- for (String value : compoundValue.split(quotedValue)) {
- result.add(decode(value));
+ for (String val : compoundValue.split(quotedValue)) {
+ result.add(decode(val));
}
}
return result;
@@ -298,6 +300,7 @@ public enum Separator {
* Splits the source array into multiple array segments using this separator,
* up to a maximum of count items. This will naturally produce copied byte
* arrays for each of the split segments.
+ *
* @param source to be split
* @param limit on how many segments are supposed to be returned. A
* non-positive value indicates no limit on number of segments.
@@ -311,6 +314,7 @@ public enum Separator {
* Splits the source array into multiple array segments using this separator,
* as many times as splits are found. This will naturally produce copied byte
* arrays for each of the split segments.
+ *
* @param source to be split
* @return source split by this separator.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
index bbf498a..8e6c259 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineHBaseSchemaConstants.java
@@ -24,11 +24,13 @@ import org.apache.hadoop.hbase.util.Bytes;
/**
* contains the constants used in the context of schema accesses for
* {@link org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity}
- * information
+ * information.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class TimelineHBaseSchemaConstants {
+public final class TimelineHBaseSchemaConstants {
+ private TimelineHBaseSchemaConstants() {
+ }
/**
* Used to create a pre-split for tables starting with a username in the
@@ -36,27 +38,28 @@ public class TimelineHBaseSchemaConstants {
* separators) so that different installations can presplit based on their own
* commonly occurring names.
*/
- private final static byte[][] USERNAME_SPLITS = { Bytes.toBytes("a"),
- Bytes.toBytes("ad"), Bytes.toBytes("an"), Bytes.toBytes("b"),
- Bytes.toBytes("ca"), Bytes.toBytes("cl"), Bytes.toBytes("d"),
- Bytes.toBytes("e"), Bytes.toBytes("f"), Bytes.toBytes("g"),
- Bytes.toBytes("h"), Bytes.toBytes("i"), Bytes.toBytes("j"),
- Bytes.toBytes("k"), Bytes.toBytes("l"), Bytes.toBytes("m"),
- Bytes.toBytes("n"), Bytes.toBytes("o"), Bytes.toBytes("q"),
- Bytes.toBytes("r"), Bytes.toBytes("s"), Bytes.toBytes("se"),
- Bytes.toBytes("t"), Bytes.toBytes("u"), Bytes.toBytes("v"),
- Bytes.toBytes("w"), Bytes.toBytes("x"), Bytes.toBytes("y"),
- Bytes.toBytes("z") };
+ private final static byte[][] USERNAME_SPLITS = {
+ Bytes.toBytes("a"), Bytes.toBytes("ad"), Bytes.toBytes("an"),
+ Bytes.toBytes("b"), Bytes.toBytes("ca"), Bytes.toBytes("cl"),
+ Bytes.toBytes("d"), Bytes.toBytes("e"), Bytes.toBytes("f"),
+ Bytes.toBytes("g"), Bytes.toBytes("h"), Bytes.toBytes("i"),
+ Bytes.toBytes("j"), Bytes.toBytes("k"), Bytes.toBytes("l"),
+ Bytes.toBytes("m"), Bytes.toBytes("n"), Bytes.toBytes("o"),
+ Bytes.toBytes("q"), Bytes.toBytes("r"), Bytes.toBytes("s"),
+ Bytes.toBytes("se"), Bytes.toBytes("t"), Bytes.toBytes("u"),
+ Bytes.toBytes("v"), Bytes.toBytes("w"), Bytes.toBytes("x"),
+ Bytes.toBytes("y"), Bytes.toBytes("z")
+ };
/**
- * The length at which keys auto-split
+ * The length at which keys auto-split.
*/
public static final String USERNAME_SPLIT_KEY_PREFIX_LENGTH = "4";
/**
* @return splits for splits where a user is a prefix.
*/
- public final static byte[][] getUsernameSplits() {
+ public static byte[][] getUsernameSplits() {
byte[][] kloon = USERNAME_SPLITS.clone();
// Deep copy.
for (int row = 0; row < USERNAME_SPLITS.length; row++) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index e30f699..2328bba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -45,15 +45,17 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
*/
@Public
@Unstable
-public class TimelineStorageUtils {
+public final class TimelineStorageUtils {
+ private TimelineStorageUtils() {
+ }
- /** empty bytes */
+ /** empty bytes. */
public static final byte[] EMPTY_BYTES = new byte[0];
- /** indicator for no limits for splitting */
+ /** indicator for no limits for splitting. */
public static final int NO_LIMIT_SPLIT = -1;
- /** milliseconds in one day */
+ /** milliseconds in one day. */
public static final long MILLIS_ONE_DAY = 86400000L;
/**
@@ -62,9 +64,9 @@ public class TimelineStorageUtils {
* copied byte arrays for each of the split segments. To identify the split
* ranges without the array copies, see {@link #splitRanges(byte[], byte[])}.
*
- * @param source
- * @param separator
- * @return byte[] array after splitting the source
+ * @param source Source array.
+ * @param separator Separator represented as a byte array.
+ * @return byte[][] after splitting the source
*/
public static byte[][] split(byte[] source, byte[] separator) {
return split(source, separator, NO_LIMIT_SPLIT);
@@ -76,10 +78,10 @@ public class TimelineStorageUtils {
* copied byte arrays for each of the split segments. To identify the split
* ranges without the array copies, see {@link #splitRanges(byte[], byte[])}.
*
- * @param source
- * @param separator
+ * @param source Source array.
+ * @param separator Separator represented as a byte array.
* @param limit a non-positive value indicates no limit on number of segments.
- * @return byte[][] after splitting the input source
+ * @return byte[][] after splitting the input source.
*/
public static byte[][] split(byte[] source, byte[] separator, int limit) {
List<Range> segments = splitRanges(source, separator, limit);
@@ -100,6 +102,10 @@ public class TimelineStorageUtils {
* Returns a list of ranges identifying [start, end) -- closed, open --
* positions within the source byte array that would be split using the
* separator byte array.
+ *
+ * @param source Source array.
+ * @param separator Separator represented as a byte array.
+ * @return a list of ranges.
*/
public static List<Range> splitRanges(byte[] source, byte[] separator) {
return splitRanges(source, separator, NO_LIMIT_SPLIT);
@@ -113,6 +119,7 @@ public class TimelineStorageUtils {
* @param source the source data
* @param separator the separator pattern to look for
* @param limit the maximum number of splits to identify in the source
+ * @return a list of ranges.
*/
public static List<Range> splitRanges(byte[] source, byte[] separator,
int limit) {
@@ -132,7 +139,7 @@ public class TimelineStorageUtils {
// everything else goes in one final segment
break;
}
- segments.add(new Range(start, i));
+ segments.add(new Range(start, i));
start = i + separator.length;
// i will be incremented again in outer for loop
i += separator.length - 1;
@@ -219,9 +226,9 @@ public class TimelineStorageUtils {
/**
* returns the timestamp of that day's start (which is midnight 00:00:00 AM)
- * for a given input timestamp
+ * for a given input timestamp.
*
- * @param ts
+ * @param ts Timestamp.
* @return timestamp of that day's beginning (midnight)
*/
public static long getTopOfTheDayTimestamp(long ts) {
@@ -233,9 +240,9 @@ public class TimelineStorageUtils {
* Combines the input array of attributes and the input aggregation operation
* into a new array of attributes.
*
- * @param attributes
- * @param aggOp
- * @return array of combined attributes
+ * @param attributes Attributes to be combined.
+ * @param aggOp Aggregation operation.
+ * @return array of combined attributes.
*/
public static Attribute[] combineAttributes(Attribute[] attributes,
AggregationOperation aggOp) {
@@ -257,8 +264,8 @@ public class TimelineStorageUtils {
* Returns a number for the new array size. The new array is the combination
* of input array of attributes and the input aggregation operation.
*
- * @param attributes
- * @param aggOp
+ * @param attributes Attributes.
+ * @param aggOp Aggregation operation.
* @return the size for the new array
*/
private static int getNewLengthCombinedAttributes(Attribute[] attributes,
@@ -283,16 +290,17 @@ public class TimelineStorageUtils {
}
/**
- * checks if an application has finished
+ * checks if an application has finished.
*
- * @param te
+ * @param te TimlineEntity object.
* @return true if application has finished else false
*/
public static boolean isApplicationFinished(TimelineEntity te) {
SortedSet<TimelineEvent> allEvents = te.getEvents();
if ((allEvents != null) && (allEvents.size() > 0)) {
TimelineEvent event = allEvents.last();
- if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ if (event.getId().equals(
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
return true;
}
}
@@ -300,26 +308,27 @@ public class TimelineStorageUtils {
}
/**
- * get the time at which an app finished
+ * get the time at which an app finished.
*
- * @param te
+ * @param te TimelineEntity object.
* @return true if application has finished else false
*/
public static long getApplicationFinishedTime(TimelineEntity te) {
SortedSet<TimelineEvent> allEvents = te.getEvents();
if ((allEvents != null) && (allEvents.size() > 0)) {
TimelineEvent event = allEvents.last();
- if (event.getId().equals(ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
+ if (event.getId().equals(
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
return event.getTimestamp();
}
}
- return 0l;
+ return 0L;
}
/**
* Checks if the input TimelineEntity object is an ApplicationEntity.
*
- * @param te
+ * @param te TimelineEntity object.
* @return true if input is an ApplicationEntity, false otherwise
*/
public static boolean isApplicationEntity(TimelineEntity te) {
@@ -329,7 +338,7 @@ public class TimelineStorageUtils {
/**
* Checks for the APPLICATION_CREATED event.
*
- * @param te
+ * @param te TimelineEntity object.
* @return true is application event exists, false otherwise
*/
public static boolean isApplicationCreated(TimelineEntity te) {
@@ -346,9 +355,9 @@ public class TimelineStorageUtils {
/**
* Returns the first seen aggregation operation as seen in the list of input
- * tags or null otherwise
+ * tags or null otherwise.
*
- * @param tags
+ * @param tags list of HBase tags.
* @return AggregationOperation
*/
public static AggregationOperation getAggregationOperationFromTagsList(
@@ -366,8 +375,8 @@ public class TimelineStorageUtils {
/**
* Creates a {@link Tag} from the input attribute.
*
- * @param attribute
- * @return Tag
+ * @param attribute Attribute from which tag has to be fetched.
+ * @return a HBase Tag.
*/
public static Tag getTagFromAttribute(Entry<String, byte[]> attribute) {
// attribute could be either an Aggregation Operation or
@@ -380,8 +389,9 @@ public class TimelineStorageUtils {
return t;
}
- AggregationCompactionDimension aggCompactDim = AggregationCompactionDimension
- .getAggregationCompactionDimension(attribute.getKey());
+ AggregationCompactionDimension aggCompactDim =
+ AggregationCompactionDimension.getAggregationCompactionDimension(
+ attribute.getKey());
if (aggCompactDim != null) {
Tag t = new Tag(aggCompactDim.getTagType(), attribute.getValue());
return t;
@@ -475,7 +485,8 @@ public class TimelineStorageUtils {
/**
* Checks if passed object is of integral type(Short/Integer/Long).
- * @param obj
+ *
+ * @param obj Object to be checked.
* @return true if object passed is of type Short or Integer or Long, false
* otherwise.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
index 555b64e..7238efa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimestampGenerator.java
@@ -40,6 +40,8 @@ public class TimestampGenerator {
/**
* Returns the current wall clock time in milliseconds, multiplied by the
* required precision.
+ *
+ * @return current timestamp.
*/
public long currentTime() {
// We want to align cell timestamps with current time.
@@ -58,6 +60,8 @@ public class TimestampGenerator {
* sustained rate of more than 1M hbase writes per second AND if region fails
* over within that time range of timestamps being generated then there may be
* collisions writing to a cell version of the same column.
+ *
+ * @return unique timestamp.
*/
public long getUniqueTimestamp() {
long lastTs;
@@ -78,8 +82,8 @@ public class TimestampGenerator {
* column at the same time, then say appId of 1001 will overlap with appId of
* 001 and there may be collisions for that flow run's specific column.
*
- * @param incomingTS
- * @param appId
+ * @param incomingTS Timestamp to be converted.
+ * @param appId Application Id.
* @return a timestamp multiplied with TS_MULTIPLIER and last few digits of
* application id
*/
@@ -101,9 +105,9 @@ public class TimestampGenerator {
/**
* truncates the last few digits of the timestamp which were supplemented by
- * the TimestampGenerator#getSupplementedTimestamp function
+ * the TimestampGenerator#getSupplementedTimestamp function.
*
- * @param incomingTS
+ * @param incomingTS Timestamp to be truncated.
* @return a truncated timestamp value
*/
public static long getTruncatedTimestamp(long incomingTS) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
index 2388ba5..757a6d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/ValueConverter.java
@@ -28,18 +28,20 @@ public interface ValueConverter {
/**
* Encode an object as a byte array depending on the converter implementation.
- * @param value
+ *
+ * @param value Value to be encoded.
* @return a byte array
- * @throws IOException
+ * @throws IOException if any problem is encountered while encoding.
*/
byte[] encodeValue(Object value) throws IOException;
/**
* Decode a byte array and convert it into an object depending on the
* converter implementation.
- * @param bytes
+ *
+ * @param bytes Byte array to be decoded.
* @return an object
- * @throws IOException
+ * @throws IOException if any problem is encountered while decoding.
*/
Object decodeValue(byte[] bytes) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
index deb8bd5..f47ba93 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumn.java
@@ -39,7 +39,7 @@ public enum EntityColumn implements Column<EntityTable> {
ID(EntityColumnFamily.INFO, "id"),
/**
- * The type of entity
+ * The type of entity.
*/
TYPE(EntityColumnFamily.INFO, "type"),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
index 5b71228..f3c7e7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityColumnPrefix.java
@@ -54,7 +54,7 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
INFO(EntityColumnFamily.INFO, "i"),
/**
- * Lifecycle events for an entity
+ * Lifecycle events for an entity.
*/
EVENT(EntityColumnFamily.INFO, "e"),
@@ -215,7 +215,8 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
* is to facilitate returning byte arrays of values that were not
* Strings. If they can be treated as Strings, you should use
* {@link #readResults(Result)} instead.
- * @throws IOException
+ * @throws IOException if there is any exception encountered while reading
+ * result.
*/
public Map<?, Object> readResultsHavingCompoundColumnQualifiers(Result result)
throws IOException {
@@ -277,8 +278,8 @@ public enum EntityColumnPrefix implements ColumnPrefix<EntityTable> {
for (EntityColumnPrefix ecp : EntityColumnPrefix.values()) {
// Find a match based column family and on name.
if (ecp.columnFamily.equals(columnFamily)
- && (((columnPrefix == null) && (ecp.getColumnPrefix() == null)) || (ecp
- .getColumnPrefix().equals(columnPrefix)))) {
+ && (((columnPrefix == null) && (ecp.getColumnPrefix() == null)) ||
+ (ecp.getColumnPrefix().equals(columnPrefix)))) {
return ecp;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
index 3eaa78c..04c633c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityRowKey.java
@@ -74,14 +74,14 @@ public class EntityRowKey {
/**
* Constructs a row key prefix for the entity table as follows:
- * {@code userName!clusterId!flowName!flowRunId!AppId}
+ * {@code userName!clusterId!flowName!flowRunId!AppId}.
*
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
- * @param appId
- * @return byte array with the row key prefix
+ * @param clusterId Context cluster id.
+ * @param userId User name.
+ * @param flowName Flow name.
+ * @param flowRunId Run Id for the flow.
+ * @param appId Application Id.
+ * @return byte array with the row key prefix.
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
String flowName, Long flowRunId, String appId) {
@@ -97,15 +97,17 @@ public class EntityRowKey {
/**
* Constructs a row key prefix for the entity table as follows:
- * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!}
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!}.
+ * Typically used while querying multiple entities of a particular entity
+ * type.
*
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
- * @param appId
- * @param entityType
- * @return byte array with the row key prefix
+ * @param clusterId Context cluster id.
+ * @param userId User name.
+ * @param flowName Flow name.
+ * @param flowRunId Run Id for the flow.
+ * @param appId Application Id.
+ * @param entityType Entity type.
+ * @return byte array with the row key prefix.
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
String flowName, Long flowRunId, String appId, String entityType) {
@@ -123,16 +125,17 @@ public class EntityRowKey {
/**
* Constructs a row key for the entity table as follows:
- * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}
+ * {@code userName!clusterId!flowName!flowRunId!AppId!entityType!entityId}.
+ * Typically used while querying a specific entity.
*
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
- * @param appId
- * @param entityType
- * @param entityId
- * @return byte array with the row key
+ * @param clusterId Context cluster id.
+ * @param userId User name.
+ * @param flowName Flow name.
+ * @param flowRunId Run Id for the flow.
+ * @param appId Application Id.
+ * @param entityType Entity type.
+ * @param entityId Entity Id.
+ * @return byte array with the row key.
*/
public static byte[] getRowKey(String clusterId, String userId,
String flowName, Long flowRunId, String appId, String entityType,
@@ -151,6 +154,9 @@ public class EntityRowKey {
/**
* Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey byte representation of row key.
+ * @return An <cite>EntityRowKey</cite> object.
*/
public static EntityRowKey parseRowKey(byte[] rowKey) {
byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index d7d770b..3e3e3ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -69,27 +69,27 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineHBas
* </pre>
*/
public class EntityTable extends BaseTable<EntityTable> {
- /** entity prefix */
+ /** entity prefix. */
private static final String PREFIX =
YarnConfiguration.TIMELINE_SERVICE_PREFIX + "entity";
- /** config param name that specifies the entity table name */
+ /** config param name that specifies the entity table name. */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
/**
* config param name that specifies the TTL for metrics column family in
- * entity table
+ * entity table.
*/
private static final String METRICS_TTL_CONF_NAME = PREFIX
+ ".table.metrics.ttl";
- /** default value for entity table name */
+ /** default value for entity table name. */
private static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
- /** default TTL is 30 days for metrics timeseries */
+ /** default TTL is 30 days for metrics timeseries. */
private static final int DEFAULT_METRICS_TTL = 2592000;
- /** default max number of versions */
+ /** default max number of versions. */
private static final int DEFAULT_METRICS_MAX_VERSIONS = 1000;
private static final Log LOG = LogFactory.getLog(EntityTable.class);
@@ -139,8 +139,8 @@ public class EntityTable extends BaseTable<EntityTable> {
metricsCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
metricsCF.setTimeToLive(hbaseConf.getInt(METRICS_TTL_CONF_NAME,
DEFAULT_METRICS_TTL));
- entityTableDescp
- .setRegionSplitPolicyClassName("org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
+ entityTableDescp.setRegionSplitPolicyClassName(
+ "org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy");
entityTableDescp.setValue("KeyPrefixRegionSplitPolicy.prefix_length",
TimelineHBaseSchemaConstants.USERNAME_SPLIT_KEY_PREFIX_LENGTH);
admin.createTable(entityTableDescp,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
index 26f1cc5..0e9578a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/package-info.java
@@ -16,6 +16,10 @@
* limitations under the License.
*/
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.entity
+ * contains classes related to implementation for entity table.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.yarn.server.timelineservice.storage.entity;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
index ff12c7b..4e2cf2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationCompactionDimension.java
@@ -26,7 +26,7 @@ import org.apache.hadoop.hbase.util.Bytes;
public enum AggregationCompactionDimension {
/**
- * the application id
+ * the application id.
*/
APPLICATION_ID((byte) 101);
@@ -50,8 +50,8 @@ public enum AggregationCompactionDimension {
return this.inBytes.clone();
}
- public static AggregationCompactionDimension getAggregationCompactionDimension(
- String aggCompactDimStr) {
+ public static AggregationCompactionDimension
+ getAggregationCompactionDimension(String aggCompactDimStr) {
for (AggregationCompactionDimension aggDim : AggregationCompactionDimension
.values()) {
if (aggDim.name().equals(aggCompactDimStr)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
index c635ce6..6240e81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/AggregationOperation.java
@@ -36,17 +36,17 @@ public enum AggregationOperation {
MAX((byte) 73),
/**
- * The metrics of the flow
+ * The metrics of the flow.
*/
SUM((byte) 79),
/**
- * application running
+ * application running.
*/
SUM_FINAL((byte) 83),
/**
- * compact
+ * compact.
*/
COMPACT((byte) 89);
@@ -71,8 +71,8 @@ public enum AggregationOperation {
}
/**
- * returns the AggregationOperation enum that represents that string
- * @param aggOpStr
+ * returns the AggregationOperation enum that represents that string.
+ * @param aggOpStr Aggregation operation.
* @return the AggregationOperation enum that represents that string
*/
public static AggregationOperation getAggregationOperation(String aggOpStr) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: YARN-4446. Refactor reader API for
better extensibility (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index b7804e7..a8a2ff8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -37,6 +37,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.AfterClass;
@@ -258,9 +261,10 @@ public class TestFileSystemTimelineReaderImpl {
public void testGetEntityDefaultView() throws Exception {
// If no fields are specified, entity is returned with default view i.e.
// only the id, type and created time.
- TimelineEntity result =
- reader.getEntity("user1", "cluster1", "flow1", 1L, "app1",
- "app", "id_1", null, null, null);
+ TimelineEntity result = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", "id_1"),
+ new TimelineDataToRetrieve(null, null, null));
Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
@@ -272,9 +276,10 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetEntityByClusterAndApp() throws Exception {
// Cluster and AppId should be enough to get an entity.
- TimelineEntity result =
- reader.getEntity(null, "cluster1", null, null, "app1",
- "app", "id_1", null, null, null);
+ TimelineEntity result = reader.getEntity(
+ new TimelineReaderContext("cluster1", null, null, null, "app1", "app",
+ "id_1"),
+ new TimelineDataToRetrieve(null, null, null));
Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
@@ -288,9 +293,10 @@ public class TestFileSystemTimelineReaderImpl {
public void testAppFlowMappingCsv() throws Exception {
// Test getting an entity by cluster and app where flow entry
// in app flow mapping csv has commas.
- TimelineEntity result =
- reader.getEntity(null, "cluster1", null, null, "app2",
- "app", "id_5", null, null, null);
+ TimelineEntity result = reader.getEntity(
+ new TimelineReaderContext("cluster1", null, null, null, "app2",
+ "app", "id_5"),
+ new TimelineDataToRetrieve(null, null, null));
Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_5")).toString(),
result.getIdentifier().toString());
@@ -300,10 +306,11 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetEntityCustomFields() throws Exception {
// Specified fields in addition to default view will be returned.
- TimelineEntity result =
- reader.getEntity("user1", "cluster1", "flow1", 1L,
- "app1", "app", "id_1", null, null,
- EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS));
+ TimelineEntity result = reader.getEntity(
+ new TimelineReaderContext("cluster1","user1", "flow1", 1L, "app1",
+ "app", "id_1"),
+ new TimelineDataToRetrieve(null, null,
+ EnumSet.of(Field.INFO, Field.CONFIGS, Field.METRICS)));
Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
@@ -318,9 +325,10 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetEntityAllFields() throws Exception {
// All fields of TimelineEntity will be returned.
- TimelineEntity result =
- reader.getEntity("user1", "cluster1", "flow1", 1L, "app1", "app",
- "id_1", null, null, EnumSet.of(Field.ALL));
+ TimelineEntity result = reader.getEntity(
+ new TimelineReaderContext("cluster1","user1", "flow1", 1L, "app1",
+ "app", "id_1"),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
Assert.assertEquals(
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
@@ -333,20 +341,21 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetAllEntities() throws Exception {
- Set<TimelineEntity> result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, null, null, null, null,
- null);
+ Set<TimelineEntity> result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null), new TimelineEntityFilters(),
+ new TimelineDataToRetrieve());
// All 3 entities will be returned
Assert.assertEquals(4, result.size());
}
@Test
public void testGetEntitiesWithLimit() throws Exception {
- Set<TimelineEntity> result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- 2L, null, null, null, null, null, null, null, null, null, null,
- null);
+ Set<TimelineEntity> result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(2L, null, null, null, null, null, null,
+ null, null), new TimelineDataToRetrieve());
Assert.assertEquals(2, result.size());
// Needs to be rewritten once hashcode and equals for
// TimelineEntity is implemented
@@ -357,10 +366,11 @@ public class TestFileSystemTimelineReaderImpl {
Assert.fail("Entity not sorted by created time");
}
}
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- 3L, null, null, null, null, null, null, null, null, null, null,
- null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(3L, null, null, null, null, null, null,
+ null, null), new TimelineDataToRetrieve());
// Even though 2 entities out of 4 have same created time, one entity
// is left out due to limit
Assert.assertEquals(3, result.size());
@@ -369,10 +379,12 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetEntitiesByTimeWindows() throws Exception {
// Get entities based on created time start and end time range.
- Set<TimelineEntity> result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, 1425016502030L, 1425016502060L, null, null, null, null, null,
- null, null, null, null);
+ Set<TimelineEntity> result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, 1425016502030L, 1425016502060L, null,
+ null, null, null, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(1, result.size());
// Only one entity with ID id_4 should be returned.
for (TimelineEntity entity : result) {
@@ -382,10 +394,12 @@ public class TestFileSystemTimelineReaderImpl {
}
// Get entities if only created time end is specified.
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, 1425016502010L, null, null, null, null, null, null,
- null, null, null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, 1425016502010L, null, null,
+ null, null, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(3, result.size());
for (TimelineEntity entity : result) {
if (entity.getId().equals("id_4")) {
@@ -394,10 +408,12 @@ public class TestFileSystemTimelineReaderImpl {
}
// Get entities if only created time start is specified.
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, 1425016502010L, null, null, null, null, null, null, null,
- null, null, null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, 1425016502010L, null, null, null,
+ null, null, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(1, result.size());
for (TimelineEntity entity : result) {
if (!entity.getId().equals("id_4")) {
@@ -411,10 +427,12 @@ public class TestFileSystemTimelineReaderImpl {
// Get entities based on info filters.
Map<String, Object> infoFilters = new HashMap<String, Object>();
infoFilters.put("info2", 3.5);
- Set<TimelineEntity> result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, infoFilters, null, null, null, null,
- null, null);
+ Set<TimelineEntity> result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilters,
+ null, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(1, result.size());
// Only one entity with ID id_3 should be returned.
for (TimelineEntity entity : result) {
@@ -427,10 +445,12 @@ public class TestFileSystemTimelineReaderImpl {
Map<String, String> configFilters = new HashMap<String, String>();
configFilters.put("config_1", "123");
configFilters.put("config_3", "abc");
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, configFilters, null, null, null,
- null, null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ configFilters, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(2, result.size());
for (TimelineEntity entity : result) {
if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
@@ -442,10 +462,12 @@ public class TestFileSystemTimelineReaderImpl {
Set<String> eventFilters = new HashSet<String>();
eventFilters.add("event_2");
eventFilters.add("event_4");
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, null, eventFilters, null,
- null, null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ null, eventFilters),
+ new TimelineDataToRetrieve());
Assert.assertEquals(1, result.size());
for (TimelineEntity entity : result) {
if (!entity.getId().equals("id_3")) {
@@ -456,10 +478,12 @@ public class TestFileSystemTimelineReaderImpl {
// Get entities based on metric filters.
Set<String> metricFilters = new HashSet<String>();
metricFilters.add("metric3");
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, metricFilters, null, null,
- null, null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilters, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(2, result.size());
// Two entities with IDs' id_1 and id_2 should be returned.
for (TimelineEntity entity : result) {
@@ -476,10 +500,12 @@ public class TestFileSystemTimelineReaderImpl {
Set<String> relatesToIds = new HashSet<String>();
relatesToIds.add("flow1");
relatesTo.put("flow", relatesToIds);
- Set<TimelineEntity> result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, relatesTo, null, null, null, null, null, null,
- null, null);
+ Set<TimelineEntity> result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, relatesTo, null, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(1, result.size());
// Only one entity with ID id_1 should be returned.
for (TimelineEntity entity : result) {
@@ -493,10 +519,12 @@ public class TestFileSystemTimelineReaderImpl {
Set<String> isRelatedToIds = new HashSet<String>();
isRelatedToIds.add("tid1_2");
isRelatedTo.put("type1", isRelatedToIds);
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, isRelatedTo, null, null, null, null, null,
- null, null);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, isRelatedTo, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
Assert.assertEquals(2, result.size());
// Two entities with IDs' id_1 and id_3 should be returned.
for (TimelineEntity entity : result) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index b74f5a1..4e07ecf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -49,6 +49,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
@@ -574,9 +577,11 @@ public class TestHBaseTimelineStorage {
matchMetrics(metricValues, metricMap);
// read the timeline entity using the reader this time
- TimelineEntity e1 = reader.getEntity(user, cluster, flow, runid, appId,
- entity.getType(), entity.getId(), null, null,
- EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, appId,
+ entity.getType(), entity.getId()),
+ new TimelineDataToRetrieve(
+ null, null, EnumSet.of(TimelineReader.Field.ALL)));
assertNotNull(e1);
// verify attributes
@@ -771,12 +776,15 @@ public class TestHBaseTimelineStorage {
assertEquals(16, colCount);
// read the timeline entity using the reader this time
- TimelineEntity e1 = reader.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), null, null,
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = reader.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, appName,
+ entity.getType(), entity.getId()),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext(cluster, user, flow, runid, appName,
+ entity.getType(), null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertNotNull(e1);
assertEquals(1, es1.size());
@@ -906,12 +914,14 @@ public class TestHBaseTimelineStorage {
}
// read the timeline entity using the reader this time
- TimelineEntity e1 = reader.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), null, null,
- EnumSet.of(TimelineReader.Field.ALL));
- TimelineEntity e2 = reader.getEntity(user, cluster, null, null, appName,
- entity.getType(), entity.getId(), null, null,
- EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, appName,
+ entity.getType(), entity.getId()),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ TimelineEntity e2 = reader.getEntity(
+ new TimelineReaderContext(cluster, user, null, null, appName,
+ entity.getType(), entity.getId()),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertNotNull(e1);
assertNotNull(e2);
assertEquals(e1, e2);
@@ -1012,12 +1022,15 @@ public class TestHBaseTimelineStorage {
assertEquals(1, rowCount);
// read the timeline entity using the reader this time
- TimelineEntity e1 = reader.getEntity(user, cluster, flow, runid, appName,
- entity.getType(), entity.getId(), null, null,
- EnumSet.of(TimelineReader.Field.ALL));
- Set<TimelineEntity> es1 = reader.getEntities(user, cluster, flow, runid,
- appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, appName,
+ entity.getType(), entity.getId()),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext(cluster, user, flow, runid, appName,
+ entity.getType(), null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertNotNull(e1);
assertEquals(1, es1.size());
@@ -1100,31 +1113,36 @@ public class TestHBaseTimelineStorage {
@Test
public void testReadEntities() throws Exception {
- TimelineEntity e1 = reader.getEntity("user1", "cluster1", "some_flow_name",
- 1002345678919L, "application_1231111111_1111","world", "hello", null,
- null, EnumSet.of(Field.ALL));
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", "hello"),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(1, e1.getIsRelatedToEntities().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null,
- EnumSet.of(Field.ALL));
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world",
+ null), new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertEquals(3, es1.size());
}
@Test
public void testReadEntitiesDefaultView() throws Exception {
- TimelineEntity e1 =
- reader.getEntity("user1", "cluster1", "some_flow_name", 1002345678919L,
- "application_1231111111_1111","world", "hello", null, null, null);
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", "hello"),
+ new TimelineDataToRetrieve());
assertNotNull(e1);
assertTrue(e1.getInfo().isEmpty() && e1.getConfigs().isEmpty() &&
e1.getMetrics().isEmpty() && e1.getIsRelatedToEntities().isEmpty() &&
e1.getRelatesToEntities().isEmpty());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null, null);
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve());
assertEquals(3, es1.size());
for (TimelineEntity e : es1) {
assertTrue(e.getInfo().isEmpty() && e.getConfigs().isEmpty() &&
@@ -1135,17 +1153,20 @@ public class TestHBaseTimelineStorage {
@Test
public void testReadEntitiesByFields() throws Exception {
- TimelineEntity e1 =
- reader.getEntity("user1", "cluster1", "some_flow_name", 1002345678919L,
- "application_1231111111_1111","world", "hello", null, null,
- EnumSet.of(Field.INFO, Field.CONFIGS));
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", "hello"),
+ new TimelineDataToRetrieve(
+ null, null, EnumSet.of(Field.INFO, Field.CONFIGS)));
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(0, e1.getIsRelatedToEntities().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null,
- EnumSet.of(Field.IS_RELATED_TO, Field.METRICS));
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(
+ null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS)));
assertEquals(3, es1.size());
int metricsCnt = 0;
int isRelatedToCnt = 0;
@@ -1165,14 +1186,17 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
- TimelineEntity e1 =
- reader.getEntity("user1", "cluster1", "some_flow_name", 1002345678919L,
- "application_1231111111_1111","world", "hello", list, null, null);
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", "hello"),
+ new TimelineDataToRetrieve(list, null, null));
assertNotNull(e1);
assertEquals(1, e1.getConfigs().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, list, null, null);
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(list, null, null));
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
@@ -1186,10 +1210,12 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
- Set<TimelineEntity> entities = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, confFilters, null, null, list, null,
- null);
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilters, null, null),
+ new TimelineDataToRetrieve(list, null, null));
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1203,14 +1229,17 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
- TimelineEntity e1 =
- reader.getEntity("user1", "cluster1", "some_flow_name", 1002345678919L,
- "application_1231111111_1111","world", "hello", null, list, null);
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", "hello"),
+ new TimelineDataToRetrieve(null, list, null));
assertNotNull(e1);
assertEquals(1, e1.getMetrics().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, list, null);
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, list, null));
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
@@ -1224,10 +1253,12 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
- Set<TimelineEntity> entities = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, metricFilters, null, null,
- list, null);
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext("cluster1","user1", "some_flow_name",
+ 1002345678919L, "application_1231111111_1111","world", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilters, null),
+ new TimelineDataToRetrieve(null, list, null));
assertEquals(1, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1238,33 +1269,40 @@ public class TestHBaseTimelineStorage {
@Test
public void testReadApps() throws Exception {
- TimelineEntity e1 = reader.getEntity("user1", "cluster1", "some_flow_name",
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null,
- EnumSet.of(Field.ALL));
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(1, e1.getIsRelatedToEntities().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, EnumSet.of(Field.ALL));
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
assertEquals(3, es1.size());
}
@Test
public void testReadAppsDefaultView() throws Exception {
- TimelineEntity e1 = reader.getEntity("user1", "cluster1", "some_flow_name",
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null);
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ new TimelineDataToRetrieve());
assertNotNull(e1);
assertTrue(e1.getInfo().isEmpty() && e1.getConfigs().isEmpty() &&
e1.getMetrics().isEmpty() && e1.getIsRelatedToEntities().isEmpty() &&
e1.getRelatesToEntities().isEmpty());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, null);
+ Set<TimelineEntity> es1 = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve());
assertEquals(3, es1.size());
for (TimelineEntity e : es1) {
assertTrue(e.getInfo().isEmpty() && e.getConfigs().isEmpty() &&
@@ -1275,18 +1313,22 @@ public class TestHBaseTimelineStorage {
@Test
public void testReadAppsByFields() throws Exception {
- TimelineEntity e1 = reader.getEntity("user1", "cluster1", "some_flow_name",
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null,
- EnumSet.of(Field.INFO, Field.CONFIGS));
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ new TimelineDataToRetrieve(
+ null, null, EnumSet.of(Field.INFO, Field.CONFIGS)));
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(0, e1.getIsRelatedToEntities().size());
Set<TimelineEntity> es1 = reader.getEntities(
- "user1", "cluster1", "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null,
- EnumSet.of(Field.IS_RELATED_TO, Field.METRICS));
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(
+ null, null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS)));
assertEquals(3, es1.size());
int metricsCnt = 0;
int isRelatedToCnt = 0;
@@ -1306,15 +1348,19 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
- TimelineEntity e1 = reader.getEntity("user1", "cluster1", "some_flow_name",
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
- TimelineEntityType.YARN_APPLICATION.toString(), null, list, null, null);
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ new TimelineDataToRetrieve(list, null, null));
assertNotNull(e1);
assertEquals(1, e1.getConfigs().size());
Set<TimelineEntity> es1 = reader.getEntities(
- "user1", "cluster1", "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, list, null, null);
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null) ,
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(list, null, null));
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
@@ -1329,9 +1375,12 @@ public class TestHBaseTimelineStorage {
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
Set<TimelineEntity> entities = reader.getEntities(
- "user1", "cluster1", "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, confFilters, null, null, list, null, null);
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilters, null, null),
+ new TimelineDataToRetrieve(list, null, null));
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1345,15 +1394,19 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
- TimelineEntity e1 = reader.getEntity("user1", "cluster1", "some_flow_name",
+ TimelineEntity e1 = reader.getEntity(
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
1002345678919L, "application_1111111111_2222",
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, list, null);
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ new TimelineDataToRetrieve(null, list, null));
assertNotNull(e1);
assertEquals(1, e1.getMetrics().size());
Set<TimelineEntity> es1 = reader.getEntities(
- "user1", "cluster1", "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, list, null);
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, list, null));
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
@@ -1368,9 +1421,12 @@ public class TestHBaseTimelineStorage {
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
Set<String> metricFilters = ImmutableSet.of("MAP1_SLOT_MILLIS");
Set<TimelineEntity> entities = reader.getEntities(
- "user1", "cluster1", "some_flow_name", 1002345678919L, null,
- TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, metricFilters, null, null, list, null);
+ new TimelineReaderContext("cluster1", "user1", "some_flow_name",
+ 1002345678919L, null, TimelineEntityType.YARN_APPLICATION.toString(),
+ null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilters, null),
+ new TimelineDataToRetrieve(null, list, null));
int metricCnt = 0;
assertEquals(1, entities.size());
for (TimelineEntity entity : entities) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index cb3f7ee..9161902 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -45,6 +45,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
@@ -179,10 +182,12 @@ public class TestHBaseStorageFlowActivity {
hbr.init(c1);
hbr.start();
// get the flow activity entity
- Set<TimelineEntity> entities =
- hbr.getEntities(null, cluster, null, null, null,
- TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
- null, null, null, null, null, null, null, null, null);
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, null, null, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null),
+ new TimelineEntityFilters(10L, null, null, null, null, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
assertEquals(1, entities.size());
for (TimelineEntity e : entities) {
FlowActivityEntity flowActivity = (FlowActivityEntity)e;
@@ -235,10 +240,12 @@ public class TestHBaseStorageFlowActivity {
hbr.init(c1);
hbr.start();
- Set<TimelineEntity> entities =
- hbr.getEntities(user, cluster, flow, null, null,
- TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
- null, null, null, null, null, null, null, null, null);
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null),
+ new TimelineEntityFilters(10L, null, null, null, null, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
assertEquals(1, entities.size());
for (TimelineEntity e : entities) {
FlowActivityEntity entity = (FlowActivityEntity)e;
@@ -350,10 +357,12 @@ public class TestHBaseStorageFlowActivity {
hbr.init(c1);
hbr.start();
- Set<TimelineEntity> entities =
- hbr.getEntities(null, cluster, null, null, null,
- TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
- null, null, null, null, null, null, null, null, null);
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, null, null, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null),
+ new TimelineEntityFilters(10L, null, null, null, null, null,
+ null, null, null),
+ new TimelineDataToRetrieve());
assertEquals(1, entities.size());
for (TimelineEntity e : entities) {
FlowActivityEntity flowActivity = (FlowActivityEntity)e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 57c326b..9504799 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -44,7 +44,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
@@ -183,8 +185,10 @@ public class TestHBaseStorageFlowRun {
hbr.init(c1);
hbr.start();
// get the flow run entity
- TimelineEntity entity = hbr.getEntity(user, cluster, flow, runid, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, null);
+ TimelineEntity entity = hbr.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineDataToRetrieve());
assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
FlowRunEntity flowRun = (FlowRunEntity)entity;
assertEquals(minStartTs, flowRun.getStartTime());
@@ -242,8 +246,10 @@ public class TestHBaseStorageFlowRun {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- TimelineEntity entity = hbr.getEntity(user, cluster, flow, runid, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null, null);
+ TimelineEntity entity = hbr.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineDataToRetrieve());
assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
Set<TimelineMetric> metrics = entity.getMetrics();
assertEquals(2, metrics.size());
@@ -350,9 +356,10 @@ public class TestHBaseStorageFlowRun {
TimelineFilterList metricsToRetrieve =
new TimelineFilterList(new TimelinePrefixFilter(TimelineCompareOp.EQUAL,
metric1.substring(0, metric1.indexOf("_") + 1)));
- TimelineEntity entity = hbr.getEntity(user, cluster, flow, runid, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), null, null,
- metricsToRetrieve, null);
+ TimelineEntity entity = hbr.getEntity(
+ new TimelineReaderContext(cluster, user, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineDataToRetrieve(null, metricsToRetrieve, null));
assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
Set<TimelineMetric> metrics = entity.getMetrics();
assertEquals(1, metrics.size());
@@ -373,9 +380,11 @@ public class TestHBaseStorageFlowRun {
}
}
- Set<TimelineEntity> entities = hbr.getEntities(user, cluster, flow, runid,
- null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null,
- null, null, null, null, null, null, null, metricsToRetrieve, null);
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, metricsToRetrieve, null));
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
Set<TimelineMetric> timelineMetrics = timelineEntity.getMetrics();
@@ -441,18 +450,21 @@ public class TestHBaseStorageFlowRun {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- Set<TimelineEntity> entities = hbr.getEntities(user, cluster, flow, runid,
- null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null,
- null, null, null, null, null, null, null, null, null);
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve());
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
assertEquals(0, timelineEntity.getMetrics().size());
}
- entities = hbr.getEntities(user, cluster, flow, runid,
- null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null,
- null, null, null, null, null, null, null, null,
- EnumSet.of(Field.METRICS));
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, runid, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
Set<TimelineMetric> timelineMetrics = timelineEntity.getMetrics();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and
checkstyle issues in timelineservice code (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
index d991b42..f9eb5b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnFamily.java
@@ -24,7 +24,8 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
/**
* Represents the flow run table column families.
*/
-public enum FlowActivityColumnFamily implements ColumnFamily<FlowActivityTable> {
+public enum FlowActivityColumnFamily
+ implements ColumnFamily<FlowActivityTable> {
/**
* Info column family houses known columns, specifically ones included in
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
index 21ddcc2..a5933da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityColumnPrefix.java
@@ -31,12 +31,13 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStor
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TypedBufferedMutator;
/**
- * Identifies partially qualified columns for the {@link FlowActivityTable}
+ * Identifies partially qualified columns for the {@link FlowActivityTable}.
*/
-public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable> {
+public enum FlowActivityColumnPrefix
+ implements ColumnPrefix<FlowActivityTable> {
/**
- * To store run ids of the flows
+ * To store run ids of the flows.
*/
RUN_ID(FlowActivityColumnFamily.INFO, "r", null);
@@ -162,8 +163,8 @@ public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable>
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
- public <T> NavigableMap<String, NavigableMap<Long, T>> readResultsWithTimestamps(
- Result result) throws IOException {
+ public <T> NavigableMap<String, NavigableMap<Long, T>>
+ readResultsWithTimestamps(Result result) throws IOException {
return column.readResultsWithTimestamps(result, columnPrefixBytes);
}
@@ -179,8 +180,8 @@ public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable>
public static final FlowActivityColumnPrefix columnFor(String columnPrefix) {
// Match column based on value, assume column family matches.
- for (FlowActivityColumnPrefix flowActivityColPrefix : FlowActivityColumnPrefix
- .values()) {
+ for (FlowActivityColumnPrefix flowActivityColPrefix :
+ FlowActivityColumnPrefix.values()) {
// Find a match based only on name.
if (flowActivityColPrefix.getColumnPrefix().equals(columnPrefix)) {
return flowActivityColPrefix;
@@ -209,8 +210,8 @@ public enum FlowActivityColumnPrefix implements ColumnPrefix<FlowActivityTable>
// TODO: needs unit test to confirm and need to update javadoc to explain
// null prefix case.
- for (FlowActivityColumnPrefix flowActivityColumnPrefix : FlowActivityColumnPrefix
- .values()) {
+ for (FlowActivityColumnPrefix flowActivityColumnPrefix :
+ FlowActivityColumnPrefix.values()) {
// Find a match based column family and on name.
if (flowActivityColumnPrefix.columnFamily.equals(columnFamily)
&& (((columnPrefix == null) && (flowActivityColumnPrefix
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
index a9598ef..80b3287 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -57,9 +57,9 @@ public class FlowActivityRowKey {
/**
* Constructs a row key prefix for the flow activity table as follows:
- * {@code clusterId!}
+ * {@code clusterId!}.
*
- * @param clusterId
+ * @param clusterId Cluster Id.
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId) {
@@ -68,10 +68,10 @@ public class FlowActivityRowKey {
/**
* Constructs a row key prefix for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!}
+ * {@code clusterId!dayTimestamp!}.
*
- * @param clusterId
- * @param dayTs
+ * @param clusterId Cluster Id.
+ * @param dayTs Start of the day timestamp.
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, long dayTs) {
@@ -82,12 +82,13 @@ public class FlowActivityRowKey {
/**
* Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowName}
+ * {@code clusterId!dayTimestamp!user!flowName}.
+ * Will insert into current day's record in the table. Uses current time to
+ * store top of the day timestamp.
*
- * Will insert into current day's record in the table
- * @param clusterId
- * @param userId
- * @param flowName
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
* @return byte array with the row key prefix
*/
public static byte[] getRowKey(String clusterId, String userId,
@@ -99,12 +100,12 @@ public class FlowActivityRowKey {
/**
* Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowName}
+ * {@code clusterId!dayTimestamp!user!flowName}.
*
- * @param clusterId
- * @param dayTs
- * @param userId
- * @param flowName
+ * @param clusterId Cluster Id.
+ * @param dayTs Top of the day timestamp.
+ * @param userId User Id.
+ * @param flowName Flow Name.
* @return byte array for the row key
*/
public static byte[] getRowKey(String clusterId, long dayTs, String userId,
@@ -118,6 +119,9 @@ public class FlowActivityRowKey {
/**
* Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey Byte representation of row key.
+ * @return A <cite>FlowActivityRowKey</cite> object.
*/
public static FlowActivityRowKey parseRowKey(byte[] rowKey) {
byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
index 315281f..8a0430c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityTable.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
*
* Example flow activity table record:
*
- * </pre>
+ * <pre>
* |-------------------------------------------|
* | Row key | Column Family |
* | | info |
@@ -52,19 +52,20 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
* </pre>
*/
public class FlowActivityTable extends BaseTable<FlowActivityTable> {
- /** flow activity table prefix */
+ /** flow activity table prefix. */
private static final String PREFIX =
YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowactivity";
- /** config param name that specifies the flowactivity table name */
+ /** config param name that specifies the flowactivity table name. */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
- /** default value for flowactivity table name */
- public static final String DEFAULT_TABLE_NAME = "timelineservice.flowactivity";
+ /** default value for flowactivity table name. */
+ public static final String DEFAULT_TABLE_NAME =
+ "timelineservice.flowactivity";
private static final Log LOG = LogFactory.getLog(FlowActivityTable.class);
- /** default max number of versions */
+ /** default max number of versions. */
public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
public FlowActivityTable() {
@@ -91,16 +92,16 @@ public class FlowActivityTable extends BaseTable<FlowActivityTable> {
+ " already exists.");
}
- HTableDescriptor FlowActivityTableDescp = new HTableDescriptor(table);
+ HTableDescriptor flowActivityTableDescp = new HTableDescriptor(table);
HColumnDescriptor infoCF =
new HColumnDescriptor(FlowActivityColumnFamily.INFO.getBytes());
infoCF.setBloomFilterType(BloomType.ROWCOL);
- FlowActivityTableDescp.addFamily(infoCF);
+ flowActivityTableDescp.addFamily(infoCF);
infoCF.setMinVersions(1);
infoCF.setMaxVersions(DEFAULT_METRICS_MAX_VERSIONS);
// TODO: figure the split policy before running in production
- admin.createTable(FlowActivityTableDescp);
+ admin.createTable(flowActivityTableDescp);
LOG.info("Status of table creation for " + table.getNameAsString() + "="
+ admin.tableExists(table));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
index e3bb52d..3d7c40e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunColumnPrefix.java
@@ -194,8 +194,8 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
* org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
* #readResultsWithTimestamps(org.apache.hadoop.hbase.client.Result)
*/
- public <T> NavigableMap<String, NavigableMap<Long, T>> readResultsWithTimestamps(
- Result result) throws IOException {
+ public <T> NavigableMap<String, NavigableMap<Long, T>>
+ readResultsWithTimestamps(Result result) throws IOException {
return column.readResultsWithTimestamps(result, columnPrefixBytes);
}
@@ -248,8 +248,8 @@ public enum FlowRunColumnPrefix implements ColumnPrefix<FlowRunTable> {
for (FlowRunColumnPrefix frcp : FlowRunColumnPrefix.values()) {
// Find a match based column family and on name.
if (frcp.columnFamily.equals(columnFamily)
- && (((columnPrefix == null) && (frcp.getColumnPrefix() == null)) || (frcp
- .getColumnPrefix().equals(columnPrefix)))) {
+ && (((columnPrefix == null) && (frcp.getColumnPrefix() == null)) ||
+ (frcp.getColumnPrefix().equals(columnPrefix)))) {
return frcp;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
index 1984157..9698f06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -46,6 +46,9 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+/**
+ * Coprocessor for flow run table.
+ */
public class FlowRunCoprocessor extends BaseRegionObserver {
@SuppressWarnings("unused")
@@ -53,9 +56,10 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
private HRegion region;
/**
- * generate a timestamp that is unique per row in a region this is per region
+ * generate a timestamp that is unique per row in a region this is per region.
*/
- private final TimestampGenerator timestampGenerator = new TimestampGenerator();
+ private final TimestampGenerator timestampGenerator =
+ new TimestampGenerator();
@Override
public void start(CoprocessorEnvironment e) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
index 2cd9625..0585dc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -56,11 +56,11 @@ public class FlowRunRowKey {
/**
* Constructs a row key prefix for the flow run table as follows: {
- * clusterId!userI!flowName!}
+ * clusterId!userI!flowName!}.
*
- * @param clusterId
- * @param userId
- * @param flowName
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
@@ -71,12 +71,12 @@ public class FlowRunRowKey {
/**
* Constructs a row key for the entity table as follows: {
- * clusterId!userI!flowName!Inverted Flow Run Id}
+ * clusterId!userI!flowName!Inverted Flow Run Id}.
*
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
+ * @param flowRunId Run Id for the flow name.
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String userId,
@@ -91,6 +91,9 @@ public class FlowRunRowKey {
/**
* Given the raw row key as bytes, returns the row key as an object.
+ *
+ * @param rowKey Byte representation of row key.
+ * @return A <cite>FlowRunRowKey</cite> object.
*/
public static FlowRunRowKey parseRowKey(byte[] rowKey) {
byte[][] rowKeyComponents = Separator.QUALIFIERS.split(rowKey);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
index 2682fea..547bef0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
@@ -84,19 +84,19 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
* </pre>
*/
public class FlowRunTable extends BaseTable<FlowRunTable> {
- /** entity prefix */
+ /** entity prefix. */
private static final String PREFIX =
YarnConfiguration.TIMELINE_SERVICE_PREFIX + ".flowrun";
- /** config param name that specifies the flowrun table name */
+ /** config param name that specifies the flowrun table name. */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
- /** default value for flowrun table name */
+ /** default value for flowrun table name. */
public static final String DEFAULT_TABLE_NAME = "timelineservice.flowrun";
private static final Log LOG = LogFactory.getLog(FlowRunTable.class);
- /** default max number of versions */
+ /** default max number of versions. */
public static final int DEFAULT_METRICS_MAX_VERSIONS = Integer.MAX_VALUE;
public FlowRunTable() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
index d541df0..6fefd15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -90,8 +90,8 @@ class FlowScanner implements RegionScanner, Closeable {
}
@Override
- public boolean nextRaw(List<Cell> cells, int limit) throws IOException {
- return nextInternal(cells, limit);
+ public boolean nextRaw(List<Cell> cells, int cellLimit) throws IOException {
+ return nextInternal(cells, cellLimit);
}
@Override
@@ -100,8 +100,8 @@ class FlowScanner implements RegionScanner, Closeable {
}
@Override
- public boolean next(List<Cell> cells, int limit) throws IOException {
- return nextInternal(cells, limit);
+ public boolean next(List<Cell> cells, int cellLimit) throws IOException {
+ return nextInternal(cells, cellLimit);
}
private String getAggregationCompactionDimension(List<Tag> tags) {
@@ -161,11 +161,12 @@ class FlowScanner implements RegionScanner, Closeable {
* column or returns the cell as is.
*
* @param cells
- * @param limit
+ * @param cellLimit
* @return true if next row is available for the scanner, false otherwise
* @throws IOException
*/
- private boolean nextInternal(List<Cell> cells, int limit) throws IOException {
+ private boolean nextInternal(List<Cell> cells, int cellLimit)
+ throws IOException {
Cell cell = null;
startNext();
// Loop through all the cells in this row
@@ -183,8 +184,8 @@ class FlowScanner implements RegionScanner, Closeable {
Set<String> alreadySeenAggDim = new HashSet<>();
int addedCnt = 0;
ValueConverter converter = null;
- while (((cell = peekAtNextCell(limit)) != null)
- && (limit <= 0 || addedCnt < limit)) {
+ while (((cell = peekAtNextCell(cellLimit)) != null)
+ && (cellLimit <= 0 || addedCnt < cellLimit)) {
byte[] newColumnQualifier = CellUtil.cloneQualifier(cell);
if (comp.compare(currentColumnQualifier, newColumnQualifier) != 0) {
if (converter != null && isNumericConverter(converter)) {
@@ -198,12 +199,12 @@ class FlowScanner implements RegionScanner, Closeable {
}
// No operation needs to be performed on non numeric converters.
if (!isNumericConverter(converter)) {
- nextCell(limit);
+ nextCell(cellLimit);
continue;
}
collectCells(currentColumnCells, currentAggOp, cell, alreadySeenAggDim,
(NumericValueConverter)converter);
- nextCell(limit);
+ nextCell(cellLimit);
}
if (!currentColumnCells.isEmpty()) {
emitCells(cells, currentColumnCells, currentAggOp,
@@ -220,7 +221,7 @@ class FlowScanner implements RegionScanner, Closeable {
}
/**
- * resets the parameters to an intialized state for next loop iteration
+ * resets the parameters to an intialized state for next loop iteration.
*
* @param cell
* @param currentAggOp
@@ -278,14 +279,12 @@ class FlowScanner implements RegionScanner, Closeable {
List<Tag> tags = Tag.asList(cell.getTagsArray(), cell.getTagsOffset(),
cell.getTagsLength());
String aggDim = getAggregationCompactionDimension(tags);
- if (alreadySeenAggDim.contains(aggDim)) {
- // if this agg dimension has already been seen,
- // since they show up in sorted order
- // we drop the rest which are older
- // in other words, this cell is older than previously seen cells
- // for that agg dim
- } else {
- // not seen this agg dim, hence consider this cell in our working set
+
+ // If this agg dimension has already been seen, since they show up in
+ // sorted order, we drop the rest which are older. In other words, this
+ // cell is older than previously seen cells for that agg dim.
+ if (!alreadySeenAggDim.contains(aggDim)) {
+ // Not seen this agg dim, hence consider this cell in our working set
currentColumnCells.add(cell);
alreadySeenAggDim.add(aggDim);
}
@@ -424,6 +423,8 @@ class FlowScanner implements RegionScanner, Closeable {
/**
* Returns whether or not the underlying scanner has more rows.
+ *
+ * @return true, if there are more cells to return, false otherwise.
*/
public boolean hasMore() {
return currentIndex < availableCells.size() ? true : hasMore;
@@ -434,15 +435,16 @@ class FlowScanner implements RegionScanner, Closeable {
* pointer to the next cell. This method can be called multiple times in a row
* to advance through all the available cells.
*
- * @param limit
+ * @param cellLimit
* the limit of number of cells to return if the next batch must be
* fetched by the wrapped scanner
* @return the next available cell or null if no more cells are available for
* the current row
- * @throws IOException
+ * @throws IOException if any problem is encountered while grabbing the next
+ * cell.
*/
- public Cell nextCell(int limit) throws IOException {
- Cell cell = peekAtNextCell(limit);
+ public Cell nextCell(int cellLimit) throws IOException {
+ Cell cell = peekAtNextCell(cellLimit);
if (cell != null) {
currentIndex++;
}
@@ -454,19 +456,20 @@ class FlowScanner implements RegionScanner, Closeable {
* pointer. Calling this method multiple times in a row will continue to
* return the same cell.
*
- * @param limit
+ * @param cellLimit
* the limit of number of cells to return if the next batch must be
* fetched by the wrapped scanner
* @return the next available cell or null if no more cells are available for
* the current row
- * @throws IOException
+ * @throws IOException if any problem is encountered while grabbing the next
+ * cell.
*/
- public Cell peekAtNextCell(int limit) throws IOException {
+ public Cell peekAtNextCell(int cellLimit) throws IOException {
if (currentIndex >= availableCells.size()) {
// done with current batch
availableCells.clear();
currentIndex = 0;
- hasMore = flowRunScanner.next(availableCells, limit);
+ hasMore = flowRunScanner.next(availableCells, cellLimit);
}
Cell cell = null;
if (currentIndex < availableCells.size()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
new file mode 100644
index 0000000..04963f3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.flow
+ * contains classes related to implementation for flow related tables, viz. flow
+ * run table and flow activity table.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
index f652ffd..e78db2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/package-info.java
@@ -16,6 +16,10 @@
* limitations under the License.
*/
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage contains
+ * classes which define and implement reading and writing to backend storage.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.yarn.server.timelineservice.storage;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
index 387f7d7..0de09e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/ApplicationEntityReader.java
@@ -98,7 +98,7 @@ class ApplicationEntityReader extends GenericEntityReader {
TimelineEntityFilters filters = getFilters();
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.EVENTS) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getEventFilters() == null)) {
+ (isSingleEntityRead() || filters.getEventFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -107,7 +107,7 @@ class ApplicationEntityReader extends GenericEntityReader {
// info not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.INFO) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getInfoFilters() == null)) {
+ (isSingleEntityRead() || filters.getInfoFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -116,7 +116,7 @@ class ApplicationEntityReader extends GenericEntityReader {
// is releated to not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.IS_RELATED_TO) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getIsRelatedTo() == null)) {
+ (isSingleEntityRead() || filters.getIsRelatedTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -125,7 +125,7 @@ class ApplicationEntityReader extends GenericEntityReader {
// relates to not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.RELATES_TO) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getRelatesTo() == null)) {
+ (isSingleEntityRead() || filters.getRelatesTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -133,7 +133,7 @@ class ApplicationEntityReader extends GenericEntityReader {
}
list.addFilter(infoColFamilyList);
if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS) ||
- (!singleEntityRead && filters.getConfigFilters() != null)) ||
+ (!isSingleEntityRead() && filters.getConfigFilters() != null)) ||
(dataToRetrieve.getConfsToRetrieve() != null &&
!dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty())) {
FilterList filterCfg =
@@ -148,7 +148,7 @@ class ApplicationEntityReader extends GenericEntityReader {
list.addFilter(filterCfg);
}
if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) ||
- (!singleEntityRead && filters.getMetricFilters() != null)) ||
+ (!isSingleEntityRead() && filters.getMetricFilters() != null)) ||
(dataToRetrieve.getMetricsToRetrieve() != null &&
!dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
FilterList filterMetrics =
@@ -177,7 +177,7 @@ class ApplicationEntityReader extends GenericEntityReader {
if (filterList != null && !filterList.getFilters().isEmpty()) {
get.setFilter(filterList);
}
- return table.getResult(hbaseConf, conn, get);
+ return getTable().getResult(hbaseConf, conn, get);
}
@Override
@@ -186,7 +186,7 @@ class ApplicationEntityReader extends GenericEntityReader {
"clusterId shouldn't be null");
Preconditions.checkNotNull(getContext().getEntityType(),
"entityType shouldn't be null");
- if (singleEntityRead) {
+ if (isSingleEntityRead()) {
Preconditions.checkNotNull(getContext().getAppId(),
"appId shouldn't be null");
} else {
@@ -201,14 +201,14 @@ class ApplicationEntityReader extends GenericEntityReader {
protected void augmentParams(Configuration hbaseConf, Connection conn)
throws IOException {
TimelineReaderContext context = getContext();
- if (singleEntityRead) {
+ if (isSingleEntityRead()) {
if (context.getFlowName() == null || context.getFlowRunId() == null ||
context.getUserId() == null) {
FlowContext flowContext = lookupFlowContext(
context.getClusterId(), context.getAppId(), hbaseConf, conn);
- context.setFlowName(flowContext.flowName);
- context.setFlowRunId(flowContext.flowRunId);
- context.setUserId(flowContext.userId);
+ context.setFlowName(flowContext.getFlowName());
+ context.setFlowRunId(flowContext.getFlowRunId());
+ context.setUserId(flowContext.getUserId());
}
}
getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
@@ -234,7 +234,7 @@ class ApplicationEntityReader extends GenericEntityReader {
newList.addFilter(filterList);
}
scan.setFilter(newList);
- return table.getResultScanner(hbaseConf, conn, scan);
+ return getTable().getResultScanner(hbaseConf, conn, scan);
}
@Override
@@ -252,7 +252,7 @@ class ApplicationEntityReader extends GenericEntityReader {
Number createdTime =
(Number)ApplicationColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue());
- if (!singleEntityRead &&
+ if (!isSingleEntityRead() &&
(entity.getCreatedTime() < filters.getCreatedTimeBegin() ||
entity.getCreatedTime() > filters.getCreatedTimeEnd())) {
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
index 96350da..0d2bdd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowActivityEntityReader.java
@@ -111,7 +111,7 @@ class FlowActivityEntityReader extends TimelineEntityReader {
// the scanner may still return more than the limit; therefore we need to
// read the right number as we iterate
scan.setFilter(new PageFilter(getFilters().getLimit()));
- return table.getResultScanner(hbaseConf, conn, scan);
+ return getTable().getResultScanner(hbaseConf, conn, scan);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
index 2d1c41c..743315c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/FlowRunEntityReader.java
@@ -82,7 +82,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
"userId shouldn't be null");
Preconditions.checkNotNull(getContext().getFlowName(),
"flowName shouldn't be null");
- if (singleEntityRead) {
+ if (isSingleEntityRead()) {
Preconditions.checkNotNull(getContext().getFlowRunId(),
"flowRunId shouldn't be null");
}
@@ -103,7 +103,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
new BinaryComparator(FlowRunColumnFamily.INFO.getBytes()));
TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
// Metrics not required.
- if (!singleEntityRead &&
+ if (!isSingleEntityRead() &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL)) {
FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
@@ -137,7 +137,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
if (filterList != null && !filterList.getFilters().isEmpty()) {
get.setFilter(filterList);
}
- return table.getResult(hbaseConf, conn, get);
+ return getTable().getResult(hbaseConf, conn, get);
}
@Override
@@ -154,7 +154,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
newList.addFilter(filterList);
}
scan.setFilter(newList);
- return table.getResultScanner(hbaseConf, conn, scan);
+ return getTable().getResultScanner(hbaseConf, conn, scan);
}
@Override
@@ -163,7 +163,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
FlowRunEntity flowRun = new FlowRunEntity();
flowRun.setUser(context.getUserId());
flowRun.setName(context.getFlowName());
- if (singleEntityRead) {
+ if (isSingleEntityRead()) {
flowRun.setRunId(context.getFlowRunId());
} else {
FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
@@ -175,7 +175,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
if (startTime != null) {
flowRun.setStartTime(startTime.longValue());
}
- if (!singleEntityRead &&
+ if (!isSingleEntityRead() &&
(flowRun.getStartTime() < getFilters().getCreatedTimeBegin() ||
flowRun.getStartTime() > getFilters().getCreatedTimeEnd())) {
return null;
@@ -194,7 +194,7 @@ class FlowRunEntityReader extends TimelineEntityReader {
}
// read metrics
- if (singleEntityRead ||
+ if (isSingleEntityRead() ||
getDataToRetrieve().getFieldsToRetrieve().contains(Field.METRICS)) {
readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index 3bc2f3f..d8f73d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -47,8 +47,8 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilter
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
+import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
@@ -118,7 +118,7 @@ class GenericEntityReader extends TimelineEntityReader {
// Events not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.EVENTS) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getEventFilters() == null)) {
+ (isSingleEntityRead() || filters.getEventFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -127,7 +127,7 @@ class GenericEntityReader extends TimelineEntityReader {
// info not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.INFO) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getInfoFilters() == null)) {
+ (isSingleEntityRead() || filters.getInfoFilters() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -136,7 +136,7 @@ class GenericEntityReader extends TimelineEntityReader {
// is related to not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.IS_RELATED_TO) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getIsRelatedTo() == null)) {
+ (isSingleEntityRead() || filters.getIsRelatedTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -145,7 +145,7 @@ class GenericEntityReader extends TimelineEntityReader {
// relates to not required.
if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.RELATES_TO) &&
!dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (singleEntityRead || filters.getRelatesTo() == null)) {
+ (isSingleEntityRead() || filters.getRelatesTo() == null)) {
infoColFamilyList.addFilter(
new QualifierFilter(CompareOp.NOT_EQUAL,
new BinaryPrefixComparator(
@@ -153,7 +153,7 @@ class GenericEntityReader extends TimelineEntityReader {
}
list.addFilter(infoColFamilyList);
if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS) ||
- (!singleEntityRead && filters.getConfigFilters() != null)) ||
+ (!isSingleEntityRead() && filters.getConfigFilters() != null)) ||
(dataToRetrieve.getConfsToRetrieve() != null &&
!dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty())) {
FilterList filterCfg =
@@ -167,7 +167,7 @@ class GenericEntityReader extends TimelineEntityReader {
list.addFilter(filterCfg);
}
if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) ||
- (!singleEntityRead && filters.getMetricFilters() != null)) ||
+ (!isSingleEntityRead() && filters.getMetricFilters() != null)) ||
(dataToRetrieve.getMetricsToRetrieve() != null &&
!dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
FilterList filterMetrics =
@@ -201,14 +201,23 @@ class GenericEntityReader extends TimelineEntityReader {
}
protected static class FlowContext {
- protected final String userId;
- protected final String flowName;
- protected final Long flowRunId;
+ private final String userId;
+ private final String flowName;
+ private final Long flowRunId;
public FlowContext(String user, String flowName, Long flowRunId) {
this.userId = user;
this.flowName = flowName;
this.flowRunId = flowRunId;
}
+ protected String getUserId() {
+ return userId;
+ }
+ protected String getFlowName() {
+ return flowName;
+ }
+ protected Long getFlowRunId() {
+ return flowRunId;
+ }
}
@Override
@@ -219,7 +228,7 @@ class GenericEntityReader extends TimelineEntityReader {
"appId shouldn't be null");
Preconditions.checkNotNull(getContext().getEntityType(),
"entityType shouldn't be null");
- if (singleEntityRead) {
+ if (isSingleEntityRead()) {
Preconditions.checkNotNull(getContext().getEntityId(),
"entityId shouldn't be null");
}
@@ -254,7 +263,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (filterList != null && !filterList.getFilters().isEmpty()) {
get.setFilter(filterList);
}
- return table.getResult(hbaseConf, conn, get);
+ return getTable().getResult(hbaseConf, conn, get);
}
@Override
@@ -271,7 +280,7 @@ class GenericEntityReader extends TimelineEntityReader {
if (filterList != null && !filterList.getFilters().isEmpty()) {
scan.setFilter(filterList);
}
- return table.getResultScanner(hbaseConf, conn, scan);
+ return getTable().getResultScanner(hbaseConf, conn, scan);
}
@Override
@@ -289,7 +298,7 @@ class GenericEntityReader extends TimelineEntityReader {
// fetch created time
Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue());
- if (!singleEntityRead &&
+ if (!isSingleEntityRead() &&
(entity.getCreatedTime() < filters.getCreatedTimeBegin() ||
entity.getCreatedTime() > filters.getCreatedTimeEnd())) {
return null;
@@ -401,6 +410,14 @@ class GenericEntityReader extends TimelineEntityReader {
/**
* Helper method for reading relationship.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result result from HBase.
+ * @param prefix column prefix.
+ * @param isRelatedTo if true, means relationship is to be added to
+ * isRelatedTo, otherwise its added to relatesTo.
+ * @throws IOException if any problem is encountered while reading result.
*/
protected <T> void readRelationship(
TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
@@ -421,6 +438,13 @@ class GenericEntityReader extends TimelineEntityReader {
/**
* Helper method for reading key-value pairs for either info or config.
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param entity entity to fill.
+ * @param result result from HBase.
+ * @param prefix column prefix.
+ * @param isConfig if true, means we are reading configs, otherwise info.
+ * @throws IOException if any problem is encountered while reading result.
*/
protected <T> void readKeyValuePairs(
TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
@@ -441,6 +465,12 @@ class GenericEntityReader extends TimelineEntityReader {
* is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
* if there is no info associated with the event.
*
+ * @param entity entity to fill.
+ * @param result HBase Result.
+ * @param isApplication if true, event read is for application table,
+ * otherwise its being read for entity table.
+ * @throws IOException if any problem is encountered while reading result.
+ *
* See {@link EntityTable} and {@link ApplicationTable} for a more detailed
* schema description.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
index 454c179..281e901 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix
public abstract class TimelineEntityReader {
private static final Log LOG = LogFactory.getLog(TimelineEntityReader.class);
- protected final boolean singleEntityRead;
+ private final boolean singleEntityRead;
private TimelineReaderContext context;
private TimelineDataToRetrieve dataToRetrieve;
// used only for multiple entity read mode
@@ -56,7 +56,7 @@ public abstract class TimelineEntityReader {
/**
* Main table the entity reader uses.
*/
- protected BaseTable<?> table;
+ private BaseTable<?> table;
/**
* Specifies whether keys for this table are sorted in a manner where entities
@@ -68,6 +68,13 @@ public abstract class TimelineEntityReader {
/**
* Instantiates a reader for multiple-entity reads.
+ *
+ * @param ctxt Reader context which defines the scope in which query has to be
+ * made.
+ * @param entityFilters Filters which limit the entities returned.
+ * @param toRetrieve Data to retrieve for each entity.
+ * @param sortedKeys Specifies whether key for this table are sorted or not.
+ * If sorted, entities can be retrieved by created time.
*/
protected TimelineEntityReader(TimelineReaderContext ctxt,
TimelineEntityFilters entityFilters, TimelineDataToRetrieve toRetrieve,
@@ -78,11 +85,15 @@ public abstract class TimelineEntityReader {
this.dataToRetrieve = toRetrieve;
this.filters = entityFilters;
- this.table = getTable();
+ this.setTable(getTable());
}
/**
* Instantiates a reader for single-entity reads.
+ *
+ * @param ctxt Reader context which defines the scope in which query has to be
+ * made.
+ * @param toRetrieve Data to retrieve for each entity.
*/
protected TimelineEntityReader(TimelineReaderContext ctxt,
TimelineDataToRetrieve toRetrieve) {
@@ -90,13 +101,14 @@ public abstract class TimelineEntityReader {
this.context = ctxt;
this.dataToRetrieve = toRetrieve;
- this.table = getTable();
+ this.setTable(getTable());
}
/**
* Creates a {@link FilterList} based on fields, confs and metrics to
* retrieve. This filter list will be set in Scan/Get objects to trim down
* results fetched from HBase back-end storage.
+ *
* @return a {@link FilterList} object.
*/
protected abstract FilterList constructFilterListBasedOnFields();
@@ -115,6 +127,12 @@ public abstract class TimelineEntityReader {
/**
* Reads and deserializes a single timeline entity from the HBase storage.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @return A <cite>TimelineEntity</cite> object.
+ * @throws IOException if there is any exception encountered while reading
+ * entity.
*/
public TimelineEntity readEntity(Configuration hbaseConf, Connection conn)
throws IOException {
@@ -136,6 +154,11 @@ public abstract class TimelineEntityReader {
* Reads and deserializes a set of timeline entities from the HBase storage.
* It goes through all the results available, and returns the number of
* entries as specified in the limit in the entity's natural sort order.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @return a set of <cite>TimelineEntity</cite> objects.
+ * @throws IOException if any exception is encountered while reading entities.
*/
public Set<TimelineEntity> readEntities(Configuration hbaseConf,
Connection conn) throws IOException {
@@ -170,8 +193,12 @@ public abstract class TimelineEntityReader {
/**
* Returns the main table to be used by the entity reader.
+ *
+ * @return A reference to the table.
*/
- protected abstract BaseTable<?> getTable();
+ protected BaseTable<?> getTable() {
+ return table;
+ }
/**
* Validates the required parameters to read the entities.
@@ -180,6 +207,10 @@ public abstract class TimelineEntityReader {
/**
* Sets certain parameters to defaults if the values are not provided.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @throws IOException if any exception is encountered while setting params.
*/
protected abstract void augmentParams(Configuration hbaseConf,
Connection conn) throws IOException;
@@ -187,23 +218,35 @@ public abstract class TimelineEntityReader {
/**
* Fetches a {@link Result} instance for a single-entity read.
*
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @param filterList filter list which will be applied to HBase Get.
* @return the {@link Result} instance or null if no such record is found.
+ * @throws IOException if any exception is encountered while getting result.
*/
protected abstract Result getResult(Configuration hbaseConf, Connection conn,
FilterList filterList) throws IOException;
/**
* Fetches a {@link ResultScanner} for a multi-entity read.
+ *
+ * @param hbaseConf HBase Configuration.
+ * @param conn HBase Connection.
+ * @param filterList filter list which will be applied to HBase Scan.
+ * @return the {@link ResultScanner} instance.
+ * @throws IOException if any exception is encountered while getting results.
*/
protected abstract ResultScanner getResults(Configuration hbaseConf,
Connection conn, FilterList filterList) throws IOException;
/**
- * Given a {@link Result} instance, deserializes and creates a
- * {@link TimelineEntity}.
+ * Parses the result retrieved from HBase backend and convert it into a
+ * {@link TimelineEntity} object.
*
- * @return the {@link TimelineEntity} instance, or null if the {@link Result}
- * is null or empty.
+ * @param result Single row result of a Get/Scan.
+ * @return the <cite>TimelineEntity</cite> instance or null if the entity is
+ * filtered.
+ * @throws IOException if any exception is encountered while parsing entity.
*/
protected abstract TimelineEntity parseEntity(Result result)
throws IOException;
@@ -212,6 +255,11 @@ public abstract class TimelineEntityReader {
* Helper method for reading and deserializing {@link TimelineMetric} objects
* using the specified column prefix. The timeline metrics then are added to
* the given timeline entity.
+ *
+ * @param entity {@link TimelineEntity} object.
+ * @param result {@link Result} object retrieved from backend.
+ * @param columnPrefix Metric column prefix
+ * @throws IOException if any exception is encountered while reading metrics.
*/
protected void readMetrics(TimelineEntity entity, Result result,
ColumnPrefix<?> columnPrefix) throws IOException {
@@ -229,4 +277,18 @@ public abstract class TimelineEntityReader {
entity.addMetric(metric);
}
}
+
+ /**
+ * Checks whether the reader has been created to fetch single entity or
+ * multiple entities.
+ *
+ * @return true, if query is for single entity, false otherwise.
+ */
+ public boolean isSingleEntityRead() {
+ return singleEntityRead;
+ }
+
+ protected void setTable(BaseTable<?> baseTable) {
+ this.table = baseTable;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
index f2bdacd..b2a9476 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReaderFactory.java
@@ -25,10 +25,19 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContex
/**
* Factory methods for instantiating a timeline entity reader.
*/
-public class TimelineEntityReaderFactory {
+public final class TimelineEntityReaderFactory {
+ private TimelineEntityReaderFactory() {
+ }
+
/**
* Creates a timeline entity reader instance for reading a single entity with
* the specified input.
+ *
+ * @param context Reader context which defines the scope in which query has to
+ * be made.
+ * @param dataToRetrieve Data to retrieve for each entity.
+ * @return An implementation of <cite>TimelineEntityReader</cite> object
+ * depending on entity type.
*/
public static TimelineEntityReader createSingleEntityReader(
TimelineReaderContext context, TimelineDataToRetrieve dataToRetrieve) {
@@ -51,6 +60,13 @@ public class TimelineEntityReaderFactory {
/**
* Creates a timeline entity reader instance for reading set of entities with
* the specified input and predicates.
+ *
+ * @param context Reader context which defines the scope in which query has to
+ * be made.
+ * @param filters Filters which limit the entities returned.
+ * @param dataToRetrieve Data to retrieve for each entity.
+ * @return An implementation of <cite>TimelineEntityReader</cite> object
+ * depending on entity type.
*/
public static TimelineEntityReader createMultipleEntitiesReader(
TimelineReaderContext context, TimelineEntityFilters filters,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
index 0b3fa38..9814d6d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/package-info.java
@@ -15,6 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.storage.reader
+ * contains classes used to read entities from backend based on query type.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: Rebase to latest trunk
Posted by gt...@apache.org.
Rebase to latest trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d1fc87e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d1fc87e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d1fc87e
Branch: refs/heads/YARN-2928
Commit: 2d1fc87e1e6894eefb48f8456fdc58ce32f60a45
Parents: 6173732
Author: Li Lu <gt...@apache.org>
Authored: Wed Jan 20 01:05:52 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:24:08 2016 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/pom.xml | 8 ++++++++
.../yarn/client/api/impl/TimelineClientImpl.java | 4 ++--
.../metrics/TimelineServiceV1Publisher.java | 17 +++++++++++++++--
.../metrics/TimelineServiceV2Publisher.java | 18 +++++++++++++++---
.../TestResourceTrackerService.java | 12 ++++++++----
.../hadoop-yarn-server-timelineservice/pom.xml | 6 ------
6 files changed, 48 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index f422846..db98f94 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -155,6 +155,14 @@
<artifactId>hadoop-yarn-server-tests</artifactId>
<scope>test</scope>
<type>test-jar</type>
+ <exclusions>
+ <!-- server-tests brings in an older version of joda-time via
+ timeline service -->
+ <exclusion>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index b15278c..f36167d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -335,8 +335,8 @@ public class TimelineClientImpl extends TimelineClient {
@Override
protected void serviceStart() throws Exception {
timelineWriter = createTimelineWriter(
- configuration, authUgi, client, constructResURI(getConfig(),
- getTimelineServiceAddress(), false));
+ configuration, authUgi, client,
+ constructResURI(getConfig(), timelineServiceAddress, false));
}
protected TimelineWriter createTimelineWriter(Configuration conf,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index d858a6b..ddc8a16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -87,6 +87,17 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
app.getAmNodeLabelExpression());
entityInfo.put(ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
+ if (app.getCallerContext() != null) {
+ if (app.getCallerContext().getContext() != null) {
+ entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
+ app.getCallerContext().getContext());
+ }
+ if (app.getCallerContext().getSignature() != null) {
+ entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_SIGNATURE,
+ app.getCallerContext().getSignature());
+ }
+ }
+
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
@@ -185,8 +196,10 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
appAttempt.getHost());
eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
appAttempt.getRpcPort());
- eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
- appAttempt.getMasterContainer().getId().toString());
+ if (appAttempt.getMasterContainer() != null) {
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ appAttempt.getMasterContainer().getId().toString());
+ }
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
getDispatcher().getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 1954783..1b3aa6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -117,8 +117,18 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
entity.getConfigs().put(
ApplicationMetricsConstants.APP_NODE_LABEL_EXPRESSION,
app.getAppNodeLabelExpression());
- entity.setInfo(entityInfo);
+ if (app.getCallerContext() != null) {
+ if (app.getCallerContext().getContext() != null) {
+ entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_CONTEXT,
+ app.getCallerContext().getContext());
+ }
+ if (app.getCallerContext().getSignature() != null) {
+ entityInfo.put(ApplicationMetricsConstants.YARN_APP_CALLER_SIGNATURE,
+ app.getCallerContext().getSignature());
+ }
+ }
+ entity.setInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
tEvent.setTimestamp(createdTime);
@@ -220,8 +230,10 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
appAttempt.getHost());
eventInfo.put(AppAttemptMetricsConstants.RPC_PORT_EVENT_INFO,
appAttempt.getRpcPort());
- eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
- appAttempt.getMasterContainer().getId().toString());
+ if (appAttempt.getMasterContainer() != null) {
+ eventInfo.put(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ appAttempt.getMasterContainer().getId().toString());
+ }
tEvent.setInfo(eventInfo);
entity.addEvent(tEvent);
getDispatcher().getEventHandler().handle(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index 2034262..cc9d1da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -913,8 +913,10 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
statusList.add(status1);
NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
"", System.currentTimeMillis());
- node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth,
- statusList, null, nodeHeartbeat1));
+ NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
+ statusList, null, nodeHealth, null, null, null);
+ node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus,
+ nodeHeartbeat1));
Assert.assertEquals(1, node1.getRunningApps().size());
Assert.assertEquals(app1.getApplicationId(), node1.getRunningApps().get(0));
@@ -928,8 +930,10 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
ContainerState.RUNNING, "", 0);
statusList = new ArrayList<ContainerStatus>();
statusList.add(status2);
- node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeHealth,
- statusList, null, nodeHeartbeat2));
+ nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
+ statusList, null, nodeHealth, null, null, null);
+ node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeStatus,
+ nodeHeartbeat2));
Assert.assertEquals(1, node2.getRunningApps().size());
Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d1fc87e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
index 616ed06..758feb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/pom.xml
@@ -124,12 +124,6 @@
<dependency>
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-core</artifactId>
- <exclusions>
- <exclusion>
- <groupId>joda-time</groupId>
- <artifactId>joda-time</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-4445. Unify the term flowId
and flowName in timeline v2 codebase. Contributed by Zhan Zhang.
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
index a14d2bc..2cd9625 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunRowKey.java
@@ -27,14 +27,14 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStor
public class FlowRunRowKey {
private final String clusterId;
private final String userId;
- private final String flowId;
+ private final String flowName;
private final long flowRunId;
- public FlowRunRowKey(String clusterId, String userId, String flowId,
+ public FlowRunRowKey(String clusterId, String userId, String flowName,
long flowRunId) {
this.clusterId = clusterId;
this.userId = userId;
- this.flowId = flowId;
+ this.flowName = flowName;
this.flowRunId = flowRunId;
}
@@ -46,8 +46,8 @@ public class FlowRunRowKey {
return userId;
}
- public String getFlowId() {
- return flowId;
+ public String getFlowName() {
+ return flowName;
}
public long getFlowRunId() {
@@ -56,33 +56,33 @@ public class FlowRunRowKey {
/**
* Constructs a row key prefix for the flow run table as follows: {
- * clusterId!userI!flowId!}
+ * clusterId!userI!flowName!}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @return byte array with the row key prefix
*/
public static byte[] getRowKeyPrefix(String clusterId, String userId,
- String flowId) {
+ String flowName) {
return Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId, userId,
- flowId, ""));
+ flowName, ""));
}
/**
* Constructs a row key for the entity table as follows: {
- * clusterId!userI!flowId!Inverted Flow Run Id}
+ * clusterId!userI!flowName!Inverted Flow Run Id}
*
* @param clusterId
* @param userId
- * @param flowId
+ * @param flowName
* @param flowRunId
* @return byte array with the row key
*/
public static byte[] getRowKey(String clusterId, String userId,
- String flowId, Long flowRunId) {
+ String flowName, Long flowRunId) {
byte[] first = Bytes.toBytes(Separator.QUALIFIERS.joinEncoded(clusterId,
- userId, flowId));
+ userId, flowName));
// Note that flowRunId is a long, so we can't encode them all at the same
// time.
byte[] second = Bytes.toBytes(TimelineStorageUtils.invertLong(flowRunId));
@@ -104,10 +104,10 @@ public class FlowRunRowKey {
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[0]));
String userId =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[1]));
- String flowId =
+ String flowName =
Separator.QUALIFIERS.decode(Bytes.toString(rowKeyComponents[2]));
long flowRunId =
TimelineStorageUtils.invertLong(Bytes.toLong(rowKeyComponents[3]));
- return new FlowRunRowKey(clusterId, userId, flowId, flowRunId);
+ return new FlowRunRowKey(clusterId, userId, flowName, flowRunId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
index b1b93c1..2682fea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunTable.java
@@ -47,7 +47,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
* |-------------------------------------------|
* | clusterId! | flow_version:version7 |
* | userName! | |
- * | flowId! | running_apps:1 |
+ * | flowName! | running_apps:1 |
* | flowRunId | |
* | | min_start_time:1392995080000 |
* | | #0:"" |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 91f6ee5..9c74e2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -190,7 +190,7 @@ public class TestTimelineReaderWebServices {
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/entity/cluster1/app1/app/id_1?userid=user1&" +
- "flowid=flow1&flowrunid=1");
+ "flowname=flow1&flowrunid=1");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 818cd89..0918d40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -595,7 +595,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/app/cluster1/application_1111111111_1111?" +
- "userid=user1&fields=ALL&flowid=flow_name&flowrunid=1002345678919");
+ "userid=user1&fields=ALL&flowname=flow_name&flowrunid=1002345678919");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertNotNull(entity);
@@ -613,7 +613,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/app/application_1111111111_2222?userid=user1" +
- "&fields=metrics&flowid=flow_name&flowrunid=1002345678919");
+ "&fields=metrics&flowname=flow_name&flowrunid=1002345678919");
resp = getResponse(client, uri);
entity = resp.getEntity(TimelineEntity.class);
assertNotNull(entity);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index bc7b3a4..3198307 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -841,7 +841,7 @@ public class TestHBaseTimelineStorage {
assertEquals(user, key.getUserId());
assertEquals(cluster, key.getClusterId());
- assertEquals(flow, key.getFlowId());
+ assertEquals(flow, key.getFlowName());
assertEquals(runid, key.getFlowRunId());
assertEquals(appName, key.getAppId());
assertEquals(te.getType(), key.getEntityType());
@@ -856,7 +856,7 @@ public class TestHBaseTimelineStorage {
assertEquals(cluster, key.getClusterId());
assertEquals(user, key.getUserId());
- assertEquals(flow, key.getFlowId());
+ assertEquals(flow, key.getFlowName());
assertEquals(runid, key.getFlowRunId());
assertEquals(appName, key.getAppId());
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89b4101f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 434adac..eda1e21 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -165,7 +165,7 @@ public class TestHBaseStorageFlowActivity {
assertNotNull(flowActivityRowKey);
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
- assertEquals(flow, flowActivityRowKey.getFlowId());
+ assertEquals(flow, flowActivityRowKey.getFlowName());
long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
@@ -280,7 +280,7 @@ public class TestHBaseStorageFlowActivity {
assertNotNull(flowActivityRowKey);
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
- assertEquals(flow, flowActivityRowKey.getFlowId());
+ assertEquals(flow, flowActivityRowKey.getFlowName());
long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
@@ -409,7 +409,7 @@ public class TestHBaseStorageFlowActivity {
assertNotNull(flowActivityRowKey);
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
- assertEquals(flow, flowActivityRowKey.getFlowId());
+ assertEquals(flow, flowActivityRowKey.getFlowName());
long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
.currentTimeMillis());
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: YARN-4350. TestDistributedShell
fails for V2 scenarios. (Naganarasimha G R via Varun Saxena)
Posted by gt...@apache.org.
YARN-4350. TestDistributedShell fails for V2 scenarios. (Naganarasimha G R via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/94b7fb47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/94b7fb47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/94b7fb47
Branch: refs/heads/YARN-2928
Commit: 94b7fb4707ed4e37ca6d9902aa358b5c8c83c66f
Parents: 89b4101
Author: Varun Saxena <va...@apache.org>
Authored: Sun Dec 20 02:14:54 2015 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:22:04 2016 -0700
----------------------------------------------------------------------
.../applications/distributedshell/TestDistributedShell.java | 2 ++
.../java/org/apache/hadoop/yarn/server/MiniYARNCluster.java | 5 +++--
.../java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java | 2 --
3 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b7fb47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index e35a32e..203a7fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -176,6 +176,8 @@ public class TestDistributedShell {
conf.set(YarnConfiguration.TIMELINE_SERVICE_ENTITY_GROUP_PLUGIN_CLASSES,
DistributedShellTimelinePlugin.class.getName());
} else if (timelineVersion == 2.0f) {
+ // set version to 2
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
// disable v1 timeline server since we no longer have a server here
// enable aux-service based timeline aggregators
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b7fb47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 641cef3..b8003c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.net.ServerSocketUtil;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.util.Shell;
@@ -770,8 +771,8 @@ public class MiniYARNCluster extends CompositeService {
if (!useFixedPorts) {
String hostname = MiniYARNCluster.getHostname();
conf.set(YarnConfiguration.TIMELINE_SERVICE_ADDRESS, hostname + ":0");
- conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS, hostname
- + ":0");
+ conf.set(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ hostname + ":" + ServerSocketUtil.getPort(9188, 10));
}
appHistoryServer.init(conf);
super.serviceInit(conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/94b7fb47/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
index e67a236..9226ead 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -74,8 +74,6 @@ public class TestMiniYarnCluster {
String hostname = MiniYARNCluster.getHostname();
Assert.assertEquals(hostname + ":0",
conf.get(YarnConfiguration.TIMELINE_SERVICE_ADDRESS));
- Assert.assertEquals(hostname + ":0",
- conf.get(YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS));
cluster.start();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: YARN-4700. ATS storage has one
extra record each time the RM got restarted. (Naganarasimha G R via Varun
Saxena)
Posted by gt...@apache.org.
YARN-4700. ATS storage has one extra record each time the RM got restarted. (Naganarasimha G R via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34248bde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34248bde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34248bde
Branch: refs/heads/YARN-2928
Commit: 34248bdeab608c18860777fc77ba0706a4fe7ed9
Parents: f59c743
Author: Varun Saxena <va...@apache.org>
Authored: Fri Mar 4 19:42:22 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:33:34 2016 -0700
----------------------------------------------------------------------
.../storage/HBaseTimelineWriterImpl.java | 47 +++++++++--------
.../storage/common/TimelineStorageUtils.java | 35 +++----------
.../storage/flow/FlowActivityRowKey.java | 27 +++-------
...stTimelineReaderWebServicesHBaseStorage.java | 25 +++++----
.../storage/flow/TestFlowDataGenerator.java | 22 ++++----
.../flow/TestHBaseStorageFlowActivity.java | 53 ++++++++++++--------
.../storage/flow/TestHBaseStorageFlowRun.java | 4 +-
7 files changed, 96 insertions(+), 117 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 997b175..1afe878 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
+import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
@@ -53,11 +54,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
@@ -140,19 +141,22 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
storeRelations(rowKey, te, isApplication);
if (isApplication) {
- if (TimelineStorageUtils.isApplicationCreated(te)) {
+ TimelineEvent event = TimelineStorageUtils.getApplicationEvent(te,
+ ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ if (event != null) {
onApplicationCreated(clusterId, userId, flowName, flowVersion,
- flowRunId, appId, te);
+ flowRunId, appId, te, event.getTimestamp());
}
// if it's an application entity, store metrics
storeFlowMetricsAppRunning(clusterId, userId, flowName, flowRunId,
appId, te);
// if application has finished, store it's finish time and write final
- // values
- // of all metrics
- if (TimelineStorageUtils.isApplicationFinished(te)) {
+ // values of all metrics
+ event = TimelineStorageUtils.getApplicationEvent(te,
+ ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ if (event != null) {
onApplicationFinished(clusterId, userId, flowName, flowVersion,
- flowRunId, appId, te);
+ flowRunId, appId, te, event.getTimestamp());
}
}
}
@@ -161,7 +165,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
private void onApplicationCreated(String clusterId, String userId,
String flowName, String flowVersion, long flowRunId, String appId,
- TimelineEntity te) throws IOException {
+ TimelineEntity te, long appCreatedTimeStamp) throws IOException {
// store in App to flow table
storeInAppToFlowTable(clusterId, userId, flowName, flowRunId, appId, te);
// store in flow run table
@@ -169,7 +173,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
flowRunId, appId, te);
// store in flow activity table
storeInFlowActivityTable(clusterId, userId, flowName, flowVersion,
- flowRunId, appId, te);
+ flowRunId, appId, appCreatedTimeStamp);
}
/*
@@ -178,8 +182,9 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
*/
private void storeInFlowActivityTable(String clusterId, String userId,
String flowName, String flowVersion, long flowRunId, String appId,
- TimelineEntity te) throws IOException {
- byte[] rowKey = FlowActivityRowKey.getRowKey(clusterId, userId, flowName);
+ long activityTimeStamp) throws IOException {
+ byte[] rowKey = FlowActivityRowKey.getRowKey(clusterId, activityTimeStamp,
+ userId, flowName);
byte[] qualifier = GenericObjectMapper.write(flowRunId);
FlowActivityColumnPrefix.RUN_ID.store(rowKey, flowActivityTable, qualifier,
null, flowVersion,
@@ -214,28 +219,28 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
*/
private void onApplicationFinished(String clusterId, String userId,
String flowName, String flowVersion, long flowRunId, String appId,
- TimelineEntity te) throws IOException {
+ TimelineEntity te, long appFinishedTimeStamp) throws IOException {
// store in flow run table
storeAppFinishedInFlowRunTable(clusterId, userId, flowName, flowRunId,
- appId, te);
+ appId, te, appFinishedTimeStamp);
// indicate in the flow activity table that the app has finished
storeInFlowActivityTable(clusterId, userId, flowName, flowVersion,
- flowRunId, appId, te);
+ flowRunId, appId, appFinishedTimeStamp);
}
/*
* Update the {@link FlowRunTable} with Application Finished information
*/
private void storeAppFinishedInFlowRunTable(String clusterId, String userId,
- String flowName, long flowRunId, String appId, TimelineEntity te)
- throws IOException {
- byte[] rowKey = FlowRunRowKey.getRowKey(clusterId, userId, flowName,
- flowRunId);
- Attribute attributeAppId = AggregationCompactionDimension.APPLICATION_ID
- .getAttribute(appId);
+ String flowName, long flowRunId, String appId, TimelineEntity te,
+ long appFinishedTimeStamp) throws IOException {
+ byte[] rowKey =
+ FlowRunRowKey.getRowKey(clusterId, userId, flowName, flowRunId);
+ Attribute attributeAppId =
+ AggregationCompactionDimension.APPLICATION_ID.getAttribute(appId);
FlowRunColumn.MAX_END_TIME.store(rowKey, flowRunTable, null,
- TimelineStorageUtils.getApplicationFinishedTime(te), attributeAppId);
+ appFinishedTimeStamp, attributeAppId);
// store the final value of metrics since application has finished
Set<TimelineMetric> metrics = te.getMetrics();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index 2328bba..605dbe7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -21,9 +21,9 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedSet;
-import java.util.Map.Entry;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -308,24 +308,6 @@ public final class TimelineStorageUtils {
}
/**
- * get the time at which an app finished.
- *
- * @param te TimelineEntity object.
- * @return true if application has finished else false
- */
- public static long getApplicationFinishedTime(TimelineEntity te) {
- SortedSet<TimelineEvent> allEvents = te.getEvents();
- if ((allEvents != null) && (allEvents.size() > 0)) {
- TimelineEvent event = allEvents.last();
- if (event.getId().equals(
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE)) {
- return event.getTimestamp();
- }
- }
- return 0L;
- }
-
- /**
* Checks if the input TimelineEntity object is an ApplicationEntity.
*
* @param te TimelineEntity object.
@@ -336,21 +318,20 @@ public final class TimelineStorageUtils {
}
/**
- * Checks for the APPLICATION_CREATED event.
- *
* @param te TimelineEntity object.
- * @return true is application event exists, false otherwise
+ * @param eventId event with this id needs to be fetched
+ * @return TimelineEvent if TimelineEntity contains the desired event.
*/
- public static boolean isApplicationCreated(TimelineEntity te) {
+ public static TimelineEvent getApplicationEvent(TimelineEntity te,
+ String eventId) {
if (isApplicationEntity(te)) {
for (TimelineEvent event : te.getEvents()) {
- if (event.getId()
- .equals(ApplicationMetricsConstants.CREATED_EVENT_TYPE)) {
- return true;
+ if (event.getId().equals(eventId)) {
+ return event;
}
}
}
- return false;
+ return null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
index 80b3287..2726ae2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowActivityRowKey.java
@@ -83,36 +83,21 @@ public class FlowActivityRowKey {
/**
* Constructs a row key for the flow activity table as follows:
* {@code clusterId!dayTimestamp!user!flowName}.
- * Will insert into current day's record in the table. Uses current time to
- * store top of the day timestamp.
*
* @param clusterId Cluster Id.
- * @param userId User Id.
- * @param flowName Flow Name.
- * @return byte array with the row key prefix
- */
- public static byte[] getRowKey(String clusterId, String userId,
- String flowName) {
- long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
- .currentTimeMillis());
- return getRowKey(clusterId, dayTs, userId, flowName);
- }
-
- /**
- * Constructs a row key for the flow activity table as follows:
- * {@code clusterId!dayTimestamp!user!flowName}.
- *
- * @param clusterId Cluster Id.
- * @param dayTs Top of the day timestamp.
+ * @param eventTs event's TimeStamp.
* @param userId User Id.
* @param flowName Flow Name.
* @return byte array for the row key
*/
- public static byte[] getRowKey(String clusterId, long dayTs, String userId,
+ public static byte[] getRowKey(String clusterId, long eventTs, String userId,
String flowName) {
+ // convert it to Day's time stamp
+ eventTs = TimelineStorageUtils.getTopOfTheDayTimestamp(eventTs);
+
return Separator.QUALIFIERS.join(
Bytes.toBytes(Separator.QUALIFIERS.encode(clusterId)),
- Bytes.toBytes(TimelineStorageUtils.invertLong(dayTs)),
+ Bytes.toBytes(TimelineStorageUtils.invertLong(eventTs)),
Bytes.toBytes(Separator.QUALIFIERS.encode(userId)),
Bytes.toBytes(Separator.QUALIFIERS.encode(flowName)));
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 9eaa3de..009b488 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -62,8 +62,8 @@ import org.junit.Test;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
-import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.GenericType;
import com.sun.jersey.api.client.config.ClientConfig;
import com.sun.jersey.api.client.config.DefaultClientConfig;
import com.sun.jersey.client.urlconnection.HttpURLConnectionFactory;
@@ -128,15 +128,14 @@ public class TestTimelineReaderWebServicesHBaseStorage {
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- Long expTs = 1436512802000L;
- event.setTimestamp(expTs);
+ event.setTimestamp(cTime);
String expKey = "foo_event";
Object expVal = "test";
event.addInfo(expKey, expVal);
entity.addEvent(event);
TimelineEvent event11 = new TimelineEvent();
event11.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- expTs = 1436512802010L;
+ Long expTs = 1425019501000L;
event11.setTimestamp(expTs);
entity.addEvent(event11);
@@ -165,7 +164,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
entity1.addMetrics(metrics);
TimelineEvent event1 = new TimelineEvent();
event1.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- event1.setTimestamp(expTs);
+ event1.setTimestamp(cTime);
event1.addInfo(expKey, expVal);
entity1.addEvent(event1);
te1.addEntity(entity1);
@@ -182,7 +181,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
entity3.setCreatedTime(cTime);
TimelineEvent event2 = new TimelineEvent();
event2.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- event2.setTimestamp(1436512802037L);
+ event2.setTimestamp(cTime);
event2.addInfo("foo_event", "test");
entity3.addEvent(event2);
te3.addEntity(entity3);
@@ -196,7 +195,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
entity4.setCreatedTime(cTime);
TimelineEvent event4 = new TimelineEvent();
event4.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- event4.setTimestamp(1436512802037L);
+ event4.setTimestamp(cTime);
event4.addInfo("foo_event", "test");
entity4.addEvent(event4);
te4.addEntity(entity4);
@@ -785,10 +784,14 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertNotNull(entities);
assertEquals(1, entities.size());
+ long firstFlowActivity =
+ TimelineStorageUtils.getTopOfTheDayTimestamp(1425016501000L);
+
DateFormat fmt = TimelineReaderWebServices.DATE_FORMAT.get();
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/clusters/cluster1/flows?daterange=" + fmt.format(dayTs) +
- "-" + fmt.format(dayTs + (2*86400000L)));
+ "timeline/clusters/cluster1/flows?daterange="
+ + fmt.format(firstFlowActivity) + "-"
+ + fmt.format(dayTs));
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
assertNotNull(entities);
@@ -810,7 +813,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/flows?daterange=-" +
- fmt.format(dayTs + (2*86400000L)));
+ fmt.format(dayTs));
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
assertNotNull(entities);
@@ -818,7 +821,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/flows?daterange=" +
- fmt.format(dayTs - (2*86400000L)) + "-");
+ fmt.format(firstFlowActivity) + "-");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
assertNotNull(entities);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index a4c06f2..d45df57 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -103,8 +103,7 @@ class TestFlowDataGenerator {
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- long cTime = 20000000000000L;
- long mTime = 1425026901000L;
+ long cTime = 1425026901000L;
entity.setCreatedTime(cTime);
// add metrics
Set<TimelineMetric> metrics = new HashSet<>();
@@ -125,8 +124,7 @@ class TestFlowDataGenerator {
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- long expTs = 1436512802000L;
- event.setTimestamp(expTs);
+ event.setTimestamp(cTime);
String expKey = "foo_event";
Object expVal = "test";
event.addInfo(expKey, expVal);
@@ -134,7 +132,8 @@ class TestFlowDataGenerator {
event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
- event.setTimestamp(1436512801000L);
+ long expTs = cTime + 21600000;// start time + 6hrs
+ event.setTimestamp(expTs);
event.addInfo(expKey, expVal);
entity.addEvent(event);
@@ -149,8 +148,7 @@ class TestFlowDataGenerator {
entity.setType(type);
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- long endTs = 1439379885000L;
- event.setTimestamp(endTs);
+ event.setTimestamp(startTs);
String expKey = "foo_event_greater";
String expVal = "test_app_greater";
event.addInfo(expKey, expVal);
@@ -181,25 +179,23 @@ class TestFlowDataGenerator {
entity.setCreatedTime(startTs);
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- event.setTimestamp(System.currentTimeMillis());
+ event.setTimestamp(startTs);
entity.addEvent(event);
return entity;
}
- static TimelineEntity getFlowApp1() {
+ static TimelineEntity getFlowApp1(long appCreatedTime) {
TimelineEntity entity = new TimelineEntity();
String id = "flowActivity_test";
String type = TimelineEntityType.YARN_APPLICATION.toString();
entity.setId(id);
entity.setType(type);
- long cTime = 1425016501000L;
- entity.setCreatedTime(cTime);
+ entity.setCreatedTime(appCreatedTime);
TimelineEvent event = new TimelineEvent();
event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
- long expTs = 1436512802000L;
- event.setTimestamp(expTs);
+ event.setTimestamp(appCreatedTime);
String expKey = "foo_event";
Object expVal = "test";
event.addInfo(expKey, expVal);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index 9161902..6b23b6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -112,9 +112,9 @@ public class TestHBaseStorageFlowActivity {
String flowVersion = "CF7022C10F1354";
long runid = 1002345678919L;
String appName = "application_100000000000_1111";
- long minStartTs = 10000000000000L;
- long greaterStartTs = 30000000000000L;
- long endTs = 1439750690000L;
+ long minStartTs = 1424995200300L;
+ long greaterStartTs = 1424995200300L + 864000L;
+ long endTs = 1424995200300L + 86000000L;;
TimelineEntity entityMinStartTime = TestFlowDataGenerator
.getEntityMinStartTime(minStartTs);
@@ -155,7 +155,8 @@ public class TestHBaseStorageFlowActivity {
// check in flow activity table
Table table1 = conn.getTable(TableName
.valueOf(FlowActivityTable.DEFAULT_TABLE_NAME));
- byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
+ byte[] startRow =
+ FlowActivityRowKey.getRowKey(cluster, minStartTs, user, flow);
Get g = new Get(startRow);
Result r1 = table1.get(g);
assertNotNull(r1);
@@ -169,8 +170,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowName());
- long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
- .currentTimeMillis());
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(minStartTs);
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
checkFlowActivityRunId(runid, flowVersion, values);
@@ -216,7 +216,9 @@ public class TestHBaseStorageFlowActivity {
long runid = 1001111178919L;
TimelineEntities te = new TimelineEntities();
- TimelineEntity entityApp1 = TestFlowDataGenerator.getFlowApp1();
+ long appCreatedTime = 1425016501000L;
+ TimelineEntity entityApp1 =
+ TestFlowDataGenerator.getFlowApp1(appCreatedTime);
te.addEntity(entityApp1);
HBaseTimelineWriterImpl hbi = null;
@@ -231,7 +233,8 @@ public class TestHBaseStorageFlowActivity {
hbi.close();
}
// check flow activity
- checkFlowActivityTable(cluster, user, flow, flowVersion, runid, c1);
+ checkFlowActivityTable(cluster, user, flow, flowVersion, runid, c1,
+ appCreatedTime);
// use the reader to verify the data
HBaseTimelineReaderImpl hbr = null;
@@ -262,13 +265,16 @@ public class TestHBaseStorageFlowActivity {
}
private void checkFlowActivityTable(String cluster, String user, String flow,
- String flowVersion, long runid, Configuration c1) throws IOException {
+ String flowVersion, long runid, Configuration c1, long appCreatedTime)
+ throws IOException {
Scan s = new Scan();
s.addFamily(FlowActivityColumnFamily.INFO.getBytes());
- byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
+ byte[] startRow =
+ FlowActivityRowKey.getRowKey(cluster, appCreatedTime, user, flow);
s.setStartRow(startRow);
String clusterStop = cluster + "1";
- byte[] stopRow = FlowActivityRowKey.getRowKey(clusterStop, user, flow);
+ byte[] stopRow =
+ FlowActivityRowKey.getRowKey(clusterStop, appCreatedTime, user, flow);
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
Table table1 = conn.getTable(TableName
@@ -288,8 +294,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowName());
- long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
- .currentTimeMillis());
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(appCreatedTime);
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
assertEquals(1, values.size());
checkFlowActivityRunId(runid, flowVersion, values);
@@ -319,7 +324,9 @@ public class TestHBaseStorageFlowActivity {
long runid3 = 3333333333333L;
TimelineEntities te = new TimelineEntities();
- TimelineEntity entityApp1 = TestFlowDataGenerator.getFlowApp1();
+ long appCreatedTime = 1425016501000L;
+ TimelineEntity entityApp1 =
+ TestFlowDataGenerator.getFlowApp1(appCreatedTime);
te.addEntity(entityApp1);
HBaseTimelineWriterImpl hbi = null;
@@ -348,7 +355,7 @@ public class TestHBaseStorageFlowActivity {
}
// check flow activity
checkFlowActivityTableSeveralRuns(cluster, user, flow, c1, flowVersion1,
- runid1, flowVersion2, runid2, flowVersion3, runid3);
+ runid1, flowVersion2, runid2, flowVersion3, runid3, appCreatedTime);
// use the timeline reader to verify data
HBaseTimelineReaderImpl hbr = null;
@@ -369,8 +376,8 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivity.getCluster());
assertEquals(user, flowActivity.getUser());
assertEquals(flow, flowActivity.getFlowName());
- long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
- .currentTimeMillis());
+ long dayTs =
+ TimelineStorageUtils.getTopOfTheDayTimestamp(appCreatedTime);
assertEquals(dayTs, flowActivity.getDate().getTime());
Set<FlowRunEntity> flowRuns = flowActivity.getFlowRuns();
assertEquals(3, flowRuns.size());
@@ -395,14 +402,17 @@ public class TestHBaseStorageFlowActivity {
private void checkFlowActivityTableSeveralRuns(String cluster, String user,
String flow, Configuration c1, String flowVersion1, long runid1,
- String flowVersion2, long runid2, String flowVersion3, long runid3)
+ String flowVersion2, long runid2, String flowVersion3, long runid3,
+ long appCreatedTime)
throws IOException {
Scan s = new Scan();
s.addFamily(FlowActivityColumnFamily.INFO.getBytes());
- byte[] startRow = FlowActivityRowKey.getRowKey(cluster, user, flow);
+ byte[] startRow =
+ FlowActivityRowKey.getRowKey(cluster, appCreatedTime, user, flow);
s.setStartRow(startRow);
String clusterStop = cluster + "1";
- byte[] stopRow = FlowActivityRowKey.getRowKey(clusterStop, user, flow);
+ byte[] stopRow =
+ FlowActivityRowKey.getRowKey(clusterStop, appCreatedTime, user, flow);
s.setStopRow(stopRow);
Connection conn = ConnectionFactory.createConnection(c1);
Table table1 = conn.getTable(TableName
@@ -419,8 +429,7 @@ public class TestHBaseStorageFlowActivity {
assertEquals(cluster, flowActivityRowKey.getClusterId());
assertEquals(user, flowActivityRowKey.getUserId());
assertEquals(flow, flowActivityRowKey.getFlowName());
- long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(System
- .currentTimeMillis());
+ long dayTs = TimelineStorageUtils.getTopOfTheDayTimestamp(appCreatedTime);
assertEquals(dayTs, flowActivityRowKey.getDayTimestamp());
Map<byte[], byte[]> values = result
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34248bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 9504799..b234bfd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -52,8 +52,8 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilte
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -114,7 +114,7 @@ public class TestHBaseStorageFlowRun {
String flowVersion = "CF7022C10F1354";
long runid = 1002345678919L;
String appName = "application_100000000000_1111";
- long minStartTs = 10000000000000L;
+ long minStartTs = 1425026900000L;
long greaterStartTs = 30000000000000L;
long endTs = 1439750690000L;
TimelineEntity entityMinStartTime = TestFlowDataGenerator
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-4062. Add the flush and
compaction functionality via coprocessors and scanners for flow run table
(Vrushali C via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3157d32a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
new file mode 100644
index 0000000..ace218b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -0,0 +1,635 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertNotEquals;
+
+import java.io.IOException;
+import java.util.Map;
+import java.util.List;
+import java.util.SortedSet;
+import java.util.TreeSet;
+import java.util.ArrayList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.Tag;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.LongConverter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGenerator;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
+import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest;
+
+/**
+ * Tests the FlowRun and FlowActivity Tables
+ */
+public class TestHBaseStorageFlowRunCompaction {
+
+ private static HBaseTestingUtility util;
+
+ private final String metric1 = "MAP_SLOT_MILLIS";
+ private final String metric2 = "HDFS_BYTES_READ";
+
+ private final byte[] aRowKey = Bytes.toBytes("a");
+ private final byte[] aFamily = Bytes.toBytes("family");
+ private final byte[] aQualifier = Bytes.toBytes("qualifier");
+
+ @BeforeClass
+ public static void setupBeforeClass() throws Exception {
+ util = new HBaseTestingUtility();
+ Configuration conf = util.getConfiguration();
+ conf.setInt("hfile.format.version", 3);
+ util.startMiniCluster();
+ createSchema();
+ }
+
+ private static void createSchema() throws IOException {
+ TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
+ }
+
+ @Test
+ public void testWriteFlowRunCompaction() throws Exception {
+ String cluster = "kompaction_cluster1";
+ String user = "kompaction_FlowRun__user1";
+ String flow = "kompaction_flowRun_flow_name";
+ String flowVersion = "AF1021C19F1351";
+ long runid = 1449526652000L;
+
+ int start = 10;
+ int count = 2000;
+ int appIdSuffix = 1;
+ HBaseTimelineWriterImpl hbi = null;
+ long insertTs = System.currentTimeMillis() - count;
+ Configuration c1 = util.getConfiguration();
+ TimelineEntities te1 = null;
+ TimelineEntity entityApp1 = null;
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ // now insert count * ( 100 + 100) metrics
+ // each call to getEntityMetricsApp1 brings back 100 values
+ // of metric1 and 100 of metric2
+ for (int i = start; i < start + count; i++) {
+ String appName = "application_10240000000000_" + appIdSuffix;
+ insertTs++;
+ te1 = new TimelineEntities();
+ entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(insertTs, c1);
+ te1.addEntity(entityApp1);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
+
+ appName = "application_2048000000000_7" + appIdSuffix;
+ insertTs++;
+ te1 = new TimelineEntities();
+ entityApp1 = TestFlowDataGenerator.getEntityMetricsApp2(insertTs);
+ te1.addEntity(entityApp1);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
+ }
+ } finally {
+ String appName = "application_10240000000000_" + appIdSuffix;
+ te1 = new TimelineEntities();
+ entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1Complete(
+ insertTs + 1, c1);
+ te1.addEntity(entityApp1);
+ hbi.write(cluster, user, flow, flowVersion, runid, appName, te1);
+ hbi.flush();
+ hbi.close();
+ }
+
+ // check in flow run table
+ HRegionServer server = util.getRSForFirstRegionInTable(TableName
+ .valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
+ List<HRegion> regions = server.getOnlineRegions(TableName
+ .valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
+ assertTrue("Didn't find any regions for primary table!", regions.size() > 0);
+ // flush and compact all the regions of the primary table
+ for (HRegion region : regions) {
+ region.flushcache();
+ region.compactStores(true);
+ }
+
+ // check flow run for one flow many apps
+ checkFlowRunTable(cluster, user, flow, runid, c1, 3);
+ }
+
+
+ private void checkFlowRunTable(String cluster, String user, String flow,
+ long runid, Configuration c1, int valueCount) throws IOException {
+ Scan s = new Scan();
+ s.addFamily(FlowRunColumnFamily.INFO.getBytes());
+ byte[] startRow = FlowRunRowKey.getRowKey(cluster, user, flow, runid);
+ s.setStartRow(startRow);
+ String clusterStop = cluster + "1";
+ byte[] stopRow = FlowRunRowKey.getRowKey(clusterStop, user, flow, runid);
+ s.setStopRow(stopRow);
+ Connection conn = ConnectionFactory.createConnection(c1);
+ Table table1 = conn.getTable(TableName
+ .valueOf(FlowRunTable.DEFAULT_TABLE_NAME));
+ ResultScanner scanner = table1.getScanner(s);
+
+ int rowCount = 0;
+ for (Result result : scanner) {
+ assertNotNull(result);
+ assertTrue(!result.isEmpty());
+ Map<byte[], byte[]> values = result.getFamilyMap(FlowRunColumnFamily.INFO
+ .getBytes());
+ assertEquals(valueCount, values.size());
+
+ rowCount++;
+ // check metric1
+ byte[] q = ColumnHelper.getColumnQualifier(
+ FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric1);
+ assertTrue(values.containsKey(q));
+ assertEquals(141, Bytes.toLong(values.get(q)));
+
+ // check metric2
+ q = ColumnHelper.getColumnQualifier(
+ FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric2);
+ assertTrue(values.containsKey(q));
+ assertEquals(57, Bytes.toLong(values.get(q)));
+ }
+ assertEquals(1, rowCount);
+ }
+
+
+ private FlowScanner getFlowScannerForTestingCompaction() {
+ // create a FlowScanner object with the sole purpose of invoking a process
+ // summation;
+ CompactionRequest request = new CompactionRequest();
+ request.setIsMajor(true, true);
+ // okay to pass in nulls for the constructor arguments
+ // because all we want to do is invoke the process summation
+ FlowScanner fs = new FlowScanner(null, -1, null,
+ (request.isMajor() == true ? FlowScannerOperation.MAJOR_COMPACTION
+ : FlowScannerOperation.MINOR_COMPACTION));
+ assertNotNull(fs);
+ return fs;
+ }
+
+ @Test
+ public void checkProcessSummationMoreCellsSumFinal2()
+ throws IOException {
+ long cellValue1 = 1236L;
+ long cellValue2 = 28L;
+ long cellValue3 = 1236L;
+ long cellValue4 = 1236L;
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+
+ // note down the current timestamp
+ long currentTimestamp = System.currentTimeMillis();
+ long cell1Ts = 1200120L;
+ long cell2Ts = TimestampGenerator.getSupplementedTimestamp(
+ System.currentTimeMillis(),"application_123746661110_11202");
+ long cell3Ts = 1277719L;
+ long cell4Ts = currentTimestamp - 10;
+
+ SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+
+ List<Tag> tags = new ArrayList<>();
+ Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_1234588888_91188");
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ // create a cell with a VERY old timestamp and attribute SUM_FINAL
+ Cell c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cell1Ts, Bytes.toBytes(cellValue1), tagByteArray);
+ currentColumnCells.add(c1);
+
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_12700000001_29102");
+ tags.add(t);
+ tagByteArray = Tag.fromList(tags);
+ // create a cell with a recent timestamp and attribute SUM_FINAL
+ Cell c2 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cell2Ts, Bytes.toBytes(cellValue2), tagByteArray);
+ currentColumnCells.add(c2);
+
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM.getTagType(),
+ "application_191780000000001_8195");
+ tags.add(t);
+ tagByteArray = Tag.fromList(tags);
+ // create a cell with a VERY old timestamp but has attribute SUM
+ Cell c3 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cell3Ts, Bytes.toBytes(cellValue3), tagByteArray);
+ currentColumnCells.add(c3);
+
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM.getTagType(),
+ "application_191780000000001_98104");
+ tags.add(t);
+ tagByteArray = Tag.fromList(tags);
+ // create a cell with a VERY old timestamp but has attribute SUM
+ Cell c4 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cell4Ts, Bytes.toBytes(cellValue4), tagByteArray);
+ currentColumnCells.add(c4);
+
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+
+ // we should be getting back 4 cells
+ // one is the flow sum cell
+ // two are the cells with SUM attribute
+ // one cell with SUM_FINAL
+ assertEquals(4, cells.size());
+
+ for (int i = 0; i < cells.size(); i++) {
+ Cell returnedCell = cells.get(0);
+ assertNotNull(returnedCell);
+
+ long returnTs = returnedCell.getTimestamp();
+ long returnValue = Bytes.toLong(CellUtil
+ .cloneValue(returnedCell));
+ if (returnValue == cellValue2) {
+ assertTrue(returnTs == cell2Ts);
+ } else if (returnValue == cellValue3) {
+ assertTrue(returnTs == cell3Ts);
+ } else if (returnValue == cellValue4) {
+ assertTrue(returnTs == cell4Ts);
+ } else if (returnValue == cellValue1) {
+ assertTrue(returnTs != cell1Ts);
+ assertTrue(returnTs > cell1Ts);
+ assertTrue(returnTs >= currentTimestamp);
+ } else {
+ // raise a failure since we expect only these two values back
+ Assert.fail();
+ }
+ }
+ }
+
+ // tests with many cells
+ // of type SUM and SUM_FINAL
+ // all cells of SUM_FINAL will expire
+ @Test
+ public void checkProcessSummationMoreCellsSumFinalMany() throws IOException {
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+ int count = 200000;
+
+ long cellValueFinal = 1000L;
+ long cellValueNotFinal = 28L;
+
+ // note down the current timestamp
+ long currentTimestamp = System.currentTimeMillis();
+ long cellTsFinalStart = 10001120L;
+ long cellTsFinal = cellTsFinalStart;
+ long cellTsNotFinalStart = currentTimestamp - 5;
+ long cellTsNotFinal = cellTsNotFinalStart;
+
+ SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+ List<Tag> tags = null;
+ Tag t = null;
+ Cell c1 = null;
+
+ // insert SUM_FINAL cells
+ for (int i = 0; i < count; i++) {
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_123450000" + i + "01_19" + i);
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ // create a cell with a VERY old timestamp and attribute SUM_FINAL
+ c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cellTsFinal, Bytes.toBytes(cellValueFinal), tagByteArray);
+ currentColumnCells.add(c1);
+ cellTsFinal++;
+ }
+
+ // add SUM cells
+ for (int i = 0; i < count; i++) {
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM.getTagType(),
+ "application_1987650000" + i + "83_911" + i);
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ // create a cell with attribute SUM
+ c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cellTsNotFinal, Bytes.toBytes(cellValueNotFinal), tagByteArray);
+ currentColumnCells.add(c1);
+ cellTsNotFinal++;
+ }
+
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+
+ // we should be getting back count + 1 cells
+ // one is the flow sum cell
+ // others are the cells with SUM attribute
+ assertEquals(count + 1, cells.size());
+
+ for (int i = 0; i < cells.size(); i++) {
+ Cell returnedCell = cells.get(0);
+ assertNotNull(returnedCell);
+
+ long returnTs = returnedCell.getTimestamp();
+ long returnValue = Bytes.toLong(CellUtil
+ .cloneValue(returnedCell));
+ if (returnValue == (count * cellValueFinal)) {
+ assertTrue(returnTs > (cellTsFinalStart + count));
+ assertTrue(returnTs >= currentTimestamp);
+ } else if ((returnValue >= cellValueNotFinal)
+ && (returnValue <= cellValueNotFinal * count)) {
+ assertTrue(returnTs >= cellTsNotFinalStart);
+ assertTrue(returnTs <= cellTsNotFinalStart * count);
+ } else {
+ // raise a failure since we expect only these values back
+ Assert.fail();
+ }
+ }
+ }
+
+ // tests with many cells
+ // of type SUM and SUM_FINAL
+ // NOT cells of SUM_FINAL will expire
+ @Test
+ public void checkProcessSummationMoreCellsSumFinalVariedTags() throws IOException {
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+ int countFinal = 20100;
+ int countNotFinal = 1000;
+ int countFinalNotExpire = 7009;
+
+ long cellValueFinal = 1000L;
+ long cellValueNotFinal = 28L;
+
+ // note down the current timestamp
+ long currentTimestamp = System.currentTimeMillis();
+ long cellTsFinalStart = 10001120L;
+ long cellTsFinal = cellTsFinalStart;
+
+ long cellTsFinalStartNotExpire = TimestampGenerator.getSupplementedTimestamp(
+ System.currentTimeMillis(), "application_10266666661166_118821");
+ long cellTsFinalNotExpire = cellTsFinalStartNotExpire;
+
+ long cellTsNotFinalStart = currentTimestamp - 5;
+ long cellTsNotFinal = cellTsNotFinalStart;
+
+ SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+ List<Tag> tags = null;
+ Tag t = null;
+ Cell c1 = null;
+
+ // insert SUM_FINAL cells which will expire
+ for (int i = 0; i < countFinal; i++) {
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_123450000" + i + "01_19" + i);
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ // create a cell with a VERY old timestamp and attribute SUM_FINAL
+ c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cellTsFinal, Bytes.toBytes(cellValueFinal), tagByteArray);
+ currentColumnCells.add(c1);
+ cellTsFinal++;
+ }
+
+ // insert SUM_FINAL cells which will NOT expire
+ for (int i = 0; i < countFinalNotExpire; i++) {
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_123450000" + i + "01_19" + i);
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ // create a cell with a VERY old timestamp and attribute SUM_FINAL
+ c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cellTsFinalNotExpire, Bytes.toBytes(cellValueFinal), tagByteArray);
+ currentColumnCells.add(c1);
+ cellTsFinalNotExpire++;
+ }
+
+ // add SUM cells
+ for (int i = 0; i < countNotFinal; i++) {
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM.getTagType(),
+ "application_1987650000" + i + "83_911" + i);
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ // create a cell with attribute SUM
+ c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ cellTsNotFinal, Bytes.toBytes(cellValueNotFinal), tagByteArray);
+ currentColumnCells.add(c1);
+ cellTsNotFinal++;
+ }
+
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+
+ // we should be getting back
+ // countNotFinal + countFinalNotExpire + 1 cells
+ // one is the flow sum cell
+ // count = the cells with SUM attribute
+ // count = the cells with SUM_FINAL attribute but not expired
+ assertEquals(countFinalNotExpire + countNotFinal + 1, cells.size());
+
+ for (int i = 0; i < cells.size(); i++) {
+ Cell returnedCell = cells.get(0);
+ assertNotNull(returnedCell);
+
+ long returnTs = returnedCell.getTimestamp();
+ long returnValue = Bytes.toLong(CellUtil
+ .cloneValue(returnedCell));
+ if (returnValue == (countFinal * cellValueFinal)) {
+ assertTrue(returnTs > (cellTsFinalStart + countFinal));
+ assertTrue(returnTs >= currentTimestamp);
+ } else if (returnValue == cellValueNotFinal) {
+ assertTrue(returnTs >= cellTsNotFinalStart);
+ assertTrue(returnTs <= cellTsNotFinalStart + countNotFinal);
+ } else if (returnValue == cellValueFinal){
+ assertTrue(returnTs >= cellTsFinalStartNotExpire);
+ assertTrue(returnTs <= cellTsFinalStartNotExpire + countFinalNotExpire);
+ } else {
+ // raise a failure since we expect only these values back
+ Assert.fail();
+ }
+ }
+ }
+
+ @Test
+ public void testProcessSummationMoreCellsSumFinal() throws IOException {
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+ // note down the current timestamp
+ long currentTimestamp = System.currentTimeMillis();
+ long cellValue1 = 1236L;
+ long cellValue2 = 28L;
+
+ List<Tag> tags = new ArrayList<>();
+ Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_1234588888_999888");
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+
+ // create a cell with a VERY old timestamp and attribute SUM_FINAL
+ Cell c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ 120L, Bytes.toBytes(cellValue1), tagByteArray);
+ currentColumnCells.add(c1);
+
+ tags = new ArrayList<>();
+ t = new Tag(AggregationOperation.SUM.getTagType(),
+ "application_100000000001_119101");
+ tags.add(t);
+ tagByteArray = Tag.fromList(tags);
+
+ // create a cell with a VERY old timestamp but has attribute SUM
+ Cell c2 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ 130L, Bytes.toBytes(cellValue2), tagByteArray);
+ currentColumnCells.add(c2);
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+
+ // we should be getting back two cells
+ // one is the flow sum cell
+ // another is the cell with SUM attribute
+ assertEquals(2, cells.size());
+
+ Cell returnedCell = cells.get(0);
+ assertNotNull(returnedCell);
+ long inputTs1 = c1.getTimestamp();
+ long inputTs2 = c2.getTimestamp();
+
+ long returnTs = returnedCell.getTimestamp();
+ long returnValue = Bytes.toLong(CellUtil
+ .cloneValue(returnedCell));
+ // the returned Ts will be far greater than input ts as well as the noted
+ // current timestamp
+ if (returnValue == cellValue2) {
+ assertTrue(returnTs == inputTs2);
+ } else if (returnValue == cellValue1) {
+ assertTrue(returnTs >= currentTimestamp);
+ assertTrue(returnTs != inputTs1);
+ } else {
+ // raise a failure since we expect only these two values back
+ Assert.fail();
+ }
+ }
+
+ @Test
+ public void testProcessSummationOneCellSumFinal() throws IOException {
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+
+ // note down the current timestamp
+ long currentTimestamp = System.currentTimeMillis();
+ List<Tag> tags = new ArrayList<>();
+ Tag t = new Tag(AggregationOperation.SUM_FINAL.getTagType(),
+ "application_123458888888_999888");
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+ SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+
+ // create a cell with a VERY old timestamp
+ Cell c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ 120L, Bytes.toBytes(1110L), tagByteArray);
+ currentColumnCells.add(c1);
+
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+ // we should not get the same cell back
+ // but we get back the flow cell
+ assertEquals(1, cells.size());
+
+ Cell returnedCell = cells.get(0);
+ // it's NOT the same cell
+ assertNotEquals(c1, returnedCell);
+ long inputTs = c1.getTimestamp();
+ long returnTs = returnedCell.getTimestamp();
+ // the returned Ts will be far greater than input ts as well as the noted
+ // current timestamp
+ assertTrue(returnTs > inputTs);
+ assertTrue(returnTs >= currentTimestamp);
+ }
+
+ @Test
+ public void testProcessSummationOneCell() throws IOException {
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+
+ // note down the current timestamp
+ long currentTimestamp = System.currentTimeMillis();
+
+ // try for 1 cell with tag SUM
+ List<Tag> tags = new ArrayList<>();
+ Tag t = new Tag(AggregationOperation.SUM.getTagType(),
+ "application_123458888888_999888");
+ tags.add(t);
+ byte[] tagByteArray = Tag.fromList(tags);
+
+ SortedSet<Cell> currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+
+ Cell c1 = TimelineStorageUtils.createNewCell(aRowKey, aFamily, aQualifier,
+ currentTimestamp, Bytes.toBytes(1110L), tagByteArray);
+ currentColumnCells.add(c1);
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+ // we expect the same cell back
+ assertEquals(1, cells.size());
+ Cell c2 = cells.get(0);
+ assertEquals(c1, c2);
+ assertEquals(currentTimestamp, c2.getTimestamp());
+ }
+
+ @Test
+ public void testProcessSummationEmpty() throws IOException {
+ FlowScanner fs = getFlowScannerForTestingCompaction();
+ long currentTimestamp = System.currentTimeMillis();
+
+ SortedSet<Cell> currentColumnCells = null;
+ List<Cell> cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+ assertEquals(0, cells.size());
+
+ currentColumnCells = new TreeSet<Cell>(KeyValue.COMPARATOR);
+ cells = fs.processSummationMajorCompaction(currentColumnCells,
+ LongConverter.getInstance(), currentTimestamp);
+ assertNotNull(cells);
+ assertEquals(0, cells.size());
+ }
+
+ @AfterClass
+ public static void tearDownAfterClass() throws Exception {
+ util.shutdownMiniCluster();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: YARN-3586. RM to only get back
addresses of Collectors that NM needs to know. (Junping Du via Varun Saxena).
Posted by gt...@apache.org.
YARN-3586. RM to only get back addresses of Collectors that NM needs to know.
(Junping Du via Varun Saxena).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4a6388ff
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4a6388ff
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4a6388ff
Branch: refs/heads/YARN-2928
Commit: 4a6388ff7637b3c149574068e8956a871a8cd3e0
Parents: 94b7fb4
Author: Varun Saxena <va...@apache.org>
Authored: Tue Dec 22 20:58:54 2015 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:22:06 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/ResourceTrackerService.java | 30 +++----
.../TestResourceTrackerService.java | 82 ++++++++++++++++++++
2 files changed, 97 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6388ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 1dbbeb5..fea8183 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -26,7 +26,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@@ -525,16 +524,15 @@ public class ResourceTrackerService extends AbstractService implements
nodeHeartBeatResponse.setSystemCredentialsForApps(systemCredentials);
}
- List<ApplicationId> keepAliveApps =
- remoteNodeStatus.getKeepAliveApplications();
- if (timelineV2Enabled && keepAliveApps != null) {
+ if (timelineV2Enabled) {
// Return collectors' map that NM needs to know
- // TODO we should optimize this to only include collector info that NM
- // doesn't know yet.
- setAppCollectorsMapToResponse(keepAliveApps, nodeHeartBeatResponse);
+ setAppCollectorsMapToResponse(rmNode.getRunningApps(),
+ nodeHeartBeatResponse);
}
// 4. Send status to RMNode, saving the latest response.
+ List<ApplicationId> keepAliveApps =
+ remoteNodeStatus.getKeepAliveApplications();
RMNodeStatusEvent nodeStatusEvent =
new RMNodeStatusEvent(nodeId, remoteNodeStatus, nodeHeartBeatResponse);
if (request.getLogAggregationReportsForApps() != null
@@ -562,18 +560,20 @@ public class ResourceTrackerService extends AbstractService implements
}
private void setAppCollectorsMapToResponse(
- List<ApplicationId> liveApps, NodeHeartbeatResponse response) {
+ List<ApplicationId> runningApps, NodeHeartbeatResponse response) {
Map<ApplicationId, String> liveAppCollectorsMap = new
- ConcurrentHashMap<ApplicationId, String>();
+ HashMap<ApplicationId, String>();
Map<ApplicationId, RMApp> rmApps = rmContext.getRMApps();
- // Set collectors for all apps now.
- // TODO set collectors for only active apps running on NM (liveApps cannot be
- // used for this case)
- for (Map.Entry<ApplicationId, RMApp> rmApp : rmApps.entrySet()) {
- ApplicationId appId = rmApp.getKey();
- String appCollectorAddr = rmApp.getValue().getCollectorAddr();
+ // Set collectors for all running apps on this node.
+ for (ApplicationId appId : runningApps) {
+ String appCollectorAddr = rmApps.get(appId).getCollectorAddr();
if (appCollectorAddr != null) {
liveAppCollectorsMap.put(appId, appCollectorAddr);
+ } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Collector for applicaton: " + appId +
+ " hasn't registered yet!");
+ }
}
}
response.setAppCollectorsMap(liveAppCollectorsMap);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4a6388ff/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index f2f71ce..2034262 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -30,6 +30,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
@@ -61,6 +62,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequ
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.UnRegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
+import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NodeLabelsUtils;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
@@ -68,8 +70,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeStatusEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -862,6 +867,83 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
checkRebootedNMCount(rm, ++initialMetricCount);
}
+ @Test
+ public void testNodeHeartbeatForAppCollectorsMap() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ // set version to 2
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
+ // enable aux-service based timeline collectors
+ conf.set(YarnConfiguration.NM_AUX_SERVICES, "timeline_collector");
+ conf.set(YarnConfiguration.NM_AUX_SERVICES + "."
+ + "timeline_collector" + ".class",
+ PerNodeTimelineCollectorsAuxService.class.getName());
+
+ rm = new MockRM(conf);
+ rm.start();
+
+ MockNM nm1 = rm.registerNode("host1:1234", 5120);
+ MockNM nm2 = rm.registerNode("host2:1234", 2048);
+
+ NodeHeartbeatResponse nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ NodeHeartbeatResponse nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+
+ RMNodeImpl node1 =
+ (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ RMNodeImpl node2 =
+ (RMNodeImpl) rm.getRMContext().getRMNodes().get(nm2.getNodeId());
+
+ RMApp app1 = rm.submitApp(1024);
+ String collectorAddr1 = "1.2.3.4:5";
+ app1.setCollectorAddr(collectorAddr1);
+
+ String collectorAddr2 = "5.4.3.2:1";
+ RMApp app2 = rm.submitApp(1024);
+ app2.setCollectorAddr(collectorAddr2);
+
+ // Create a running container for app1 running on nm1
+ ContainerId runningContainerId1 = BuilderUtils.newContainerId(
+ BuilderUtils.newApplicationAttemptId(
+ app1.getApplicationId(), 0), 0);
+
+ ContainerStatus status1 = ContainerStatus.newInstance(runningContainerId1,
+ ContainerState.RUNNING, "", 0);
+ List<ContainerStatus> statusList = new ArrayList<ContainerStatus>();
+ statusList.add(status1);
+ NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
+ "", System.currentTimeMillis());
+ node1.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeHealth,
+ statusList, null, nodeHeartbeat1));
+
+ Assert.assertEquals(1, node1.getRunningApps().size());
+ Assert.assertEquals(app1.getApplicationId(), node1.getRunningApps().get(0));
+
+ // Create a running container for app2 running on nm2
+ ContainerId runningContainerId2 = BuilderUtils.newContainerId(
+ BuilderUtils.newApplicationAttemptId(
+ app2.getApplicationId(), 0), 0);
+
+ ContainerStatus status2 = ContainerStatus.newInstance(runningContainerId2,
+ ContainerState.RUNNING, "", 0);
+ statusList = new ArrayList<ContainerStatus>();
+ statusList.add(status2);
+ node2.handle(new RMNodeStatusEvent(nm2.getNodeId(), nodeHealth,
+ statusList, null, nodeHeartbeat2));
+ Assert.assertEquals(1, node2.getRunningApps().size());
+ Assert.assertEquals(app2.getApplicationId(), node2.getRunningApps().get(0));
+
+ nodeHeartbeat1 = nm1.nodeHeartbeat(true);
+ Map<ApplicationId, String> map1 = nodeHeartbeat1.getAppCollectorsMap();
+ Assert.assertEquals(1, map1.size());
+ Assert.assertEquals(collectorAddr1, map1.get(app1.getApplicationId()));
+
+ nodeHeartbeat2 = nm2.nodeHeartbeat(true);
+ Map<ApplicationId, String> map2 = nodeHeartbeat2.getAppCollectorsMap();
+ Assert.assertEquals(1, map2.size());
+ Assert.assertEquals(collectorAddr2, map2.get(app2.getApplicationId()));
+ }
+
private void checkRebootedNMCount(MockRM rm2, int count)
throws InterruptedException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: YARN-3995. Some of the NM events
are not getting published due race condition when AM container finishes in NM
(Naganarasimha G R via sjlee)
Posted by gt...@apache.org.
YARN-3995. Some of the NM events are not getting published due race condition when AM container finishes in NM (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ed5f7db9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ed5f7db9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ed5f7db9
Branch: refs/heads/YARN-2928
Commit: ed5f7db9cfe66f9eb5a70d1b1551c239436c0f23
Parents: 4a6388f
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Jan 11 10:09:34 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:22:08 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 5 ++++
.../src/main/resources/yarn-default.xml | 7 ++++++
.../PerNodeTimelineCollectorsAuxService.java | 25 +++++++++++++-------
...TestPerNodeTimelineCollectorsAuxService.java | 11 +++++----
4 files changed, 35 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index ed81eaa..8ee417b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1899,6 +1899,11 @@ public class YarnConfiguration extends Configuration {
public static final int
DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS = 60;
+ public static final String ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS =
+ TIMELINE_SERVICE_PREFIX + "app-collector.linger-period.ms";
+
+ public static final int DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS = 1000;
+
// mark app-history related configs @Private as application history is going
// to be integrated into the timeline service
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e4b562e..1867e98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2182,6 +2182,13 @@
<value>60</value>
</property>
+ <property>
+ <description>Time period till which the application collector will be alive
+ in NM, after the application master container finishes.</description>
+ <name>yarn.timeline-service.app-collector.linger-period.ms</name>
+ <value>1000</value>
+ </property>
+
<!-- Shared Cache Configuration -->
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index 0319e34..b738530 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
import java.nio.ByteBuffer;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -54,6 +57,8 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
private static final int SHUTDOWN_HOOK_PRIORITY = 30;
private final NodeTimelineCollectorManager collectorManager;
+ private long collectorLingerPeriod;
+ private ScheduledExecutorService scheduler;
public PerNodeTimelineCollectorsAuxService() {
this(new NodeTimelineCollectorManager());
@@ -70,6 +75,10 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
if (!YarnConfiguration.timelineServiceV2Enabled(conf)) {
throw new YarnException("Timeline service v2 is not enabled");
}
+ collectorLingerPeriod =
+ conf.getLong(YarnConfiguration.ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS,
+ YarnConfiguration.DEFAULT_ATS_APP_COLLECTOR_LINGER_PERIOD_IN_MS);
+ scheduler = Executors.newSingleThreadScheduledExecutor();
collectorManager.init(conf);
super.serviceInit(conf);
}
@@ -82,6 +91,12 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
@Override
protected void serviceStop() throws Exception {
+ scheduler.shutdown();
+ if (!scheduler.awaitTermination(collectorLingerPeriod,
+ TimeUnit.MILLISECONDS)) {
+ LOG.warn(
+ "Scheduler terminated before removing the application collectors");
+ }
collectorManager.stop();
super.serviceStop();
}
@@ -141,17 +156,11 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
if (context.getContainerType() == ContainerType.APPLICATION_MASTER) {
final ApplicationId appId =
context.getContainerId().getApplicationAttemptId().getApplicationId();
- new Thread(new Runnable() {
+ scheduler.schedule(new Runnable() {
public void run() {
- try {
- // TODO Temporary Fix until solution for YARN-3995 is finalized.
- Thread.sleep(1000l);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
removeApplication(appId);
}
- }).start();
+ }, collectorLingerPeriod, TimeUnit.MILLISECONDS);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ed5f7db9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
index 4fdf47e..f2775d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestPerNodeTimelineCollectorsAuxService.java
@@ -22,12 +22,14 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.mockito.Mockito.any;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.io.IOException;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Shell;
@@ -45,8 +47,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.GetTimelineCollectorCon
import org.junit.After;
import org.junit.Test;
-import java.io.IOException;
-
public class TestPerNodeTimelineCollectorsAuxService {
private ApplicationAttemptId appAttemptId;
private PerNodeTimelineCollectorsAuxService auxService;
@@ -103,8 +103,9 @@ public class TestPerNodeTimelineCollectorsAuxService {
when(context.getContainerType()).thenReturn(
ContainerType.APPLICATION_MASTER);
auxService.stopContainer(context);
-
- // TODO Temporary Fix until solution for YARN-3995 is finalized
+ // auxService should have the app's collector and need to remove only after
+ // a configured period
+ assertTrue(auxService.hasApplication(appAttemptId.getApplicationId()));
for (int i = 0; i < 4; i++) {
Thread.sleep(500l);
if (!auxService.hasApplication(appAttemptId.getApplicationId())) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-4447. Provide a mechanism to
represent complex filters and parse them at the REST layer (Varun Saxena via
sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index 57d75db..2e667d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -18,29 +18,19 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
-import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashSet;
-import java.util.Set;
import javax.servlet.http.HttpServletRequest;
+import org.apache.commons.io.IOUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
/**
* Set of utility methods to be used by timeline reader web services.
*/
final class TimelineReaderWebServicesUtils {
- private static final String COMMA_DELIMITER = ",";
- private static final String COLON_DELIMITER = ":";
private TimelineReaderWebServicesUtils() {
}
@@ -56,11 +46,10 @@ final class TimelineReaderWebServicesUtils {
* @param entityType Entity Type.
* @param entityId Entity Id.
* @return a {@link TimelineReaderContext} object.
- * @throws Exception if any problem occurs during parsing.
*/
static TimelineReaderContext createTimelineReaderContext(String clusterId,
String userId, String flowName, String flowRunId, String appId,
- String entityType, String entityId) throws Exception {
+ String entityType, String entityId) {
return new TimelineReaderContext(parseStr(clusterId), parseStr(userId),
parseStr(flowName), parseLongStr(flowRunId), parseStr(appId),
parseStr(entityType), parseStr(entityId));
@@ -79,20 +68,17 @@ final class TimelineReaderWebServicesUtils {
* @param metricfilters Entities to return must match these metric filters.
* @param eventfilters Entities to return must match these event filters.
* @return a {@link TimelineEntityFilters} object.
- * @throws Exception if any problem occurs during parsing.
+ * @throws TimelineParseException if any problem occurs during parsing.
*/
static TimelineEntityFilters createTimelineEntityFilters(String limit,
String createdTimeStart, String createdTimeEnd, String relatesTo,
String isRelatedTo, String infofilters, String conffilters,
- String metricfilters, String eventfilters) throws Exception {
+ String metricfilters, String eventfilters) throws TimelineParseException {
return new TimelineEntityFilters(parseLongStr(limit),
parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd),
- parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
- parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
- parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
- parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
- parseMetricFilters(metricfilters, COMMA_DELIMITER),
- parseValuesStr(eventfilters, COMMA_DELIMITER));
+ parseRelationFilters(relatesTo), parseRelationFilters(isRelatedTo),
+ parseKVFilters(infofilters, false), parseKVFilters(conffilters, true),
+ parseMetricFilters(metricfilters), parseEventFilters(eventfilters));
}
/**
@@ -102,12 +88,13 @@ final class TimelineReaderWebServicesUtils {
* @param metrics metrics to retrieve.
* @param fields fields to retrieve.
* @return a {@link TimelineDataToRetrieve} object.
- * @throws Exception if any problem occurs during parsing.
+ * @throws TimelineParseException if any problem occurs during parsing.
*/
static TimelineDataToRetrieve createTimelineDataToRetrieve(String confs,
- String metrics, String fields) throws Exception {
- return new TimelineDataToRetrieve(
- null, null, parseFieldsStr(fields, COMMA_DELIMITER));
+ String metrics, String fields) throws TimelineParseException {
+ return new TimelineDataToRetrieve(parseDataToRetrieve(confs),
+ parseDataToRetrieve(metrics), parseFieldsStr(
+ fields, TimelineParseConstants.COMMA_DELIMITER));
}
/**
@@ -118,110 +105,47 @@ final class TimelineReaderWebServicesUtils {
* @param delimiter string is delimited by this delimiter.
* @return set of strings.
*/
- static TimelineFilterList parseValuesStr(String str, String delimiter) {
- if (str == null || str.isEmpty()) {
- return null;
- }
- TimelineFilterList filterList = new TimelineFilterList();
- String[] strs = str.split(delimiter);
- for (String aStr : strs) {
- filterList.addFilter(new TimelineExistsFilter(TimelineCompareOp.EQUAL,
- aStr.trim()));
- }
- return filterList;
- }
-
- private static TimelineFilterList parseKeyValues(String str,
- String pairsDelim, String keyValuesDelim, boolean stringValue,
- boolean multipleValues) {
- if (str == null) {
- return null;
- }
- TimelineFilterList list = new TimelineFilterList();
- String[] pairs = str.split(pairsDelim);
- for (String pair : pairs) {
- if (pair == null || pair.trim().isEmpty()) {
- continue;
- }
- String[] pairStrs = pair.split(keyValuesDelim);
- if (pairStrs.length < 2) {
- continue;
- }
- if (!stringValue) {
- try {
- Object value =
- GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
- list.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
- pairStrs[0].trim(), value));
- } catch (IOException e) {
- list.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
- pairStrs[0].trim(), pairStrs[1].trim()));
- }
- } else {
- String key = pairStrs[0].trim();
- if (multipleValues) {
- Set<Object> values = new HashSet<Object>();
- for (int i = 1; i < pairStrs.length; i++) {
- values.add(pairStrs[i].trim());
- }
- list.addFilter(new TimelineKeyValuesFilter(
- TimelineCompareOp.EQUAL, key, values));
- } else {
- list.addFilter(new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
- key, pairStrs[1].trim()));
- }
- }
- }
- return list;
+ static TimelineFilterList parseEventFilters(String expr)
+ throws TimelineParseException {
+ return parseFilters(new TimelineParserForExistFilters(expr,
+ TimelineParseConstants.COMMA_CHAR));
}
/**
- * Parse a delimited string and convert it into a map of key-values with each
- * key having a set of values. Both the key and values are interpreted as
- * strings.
- * For instance, if pairsDelim is "," and keyValuesDelim is ":", then the
- * string should be represented as
- * "key1:value11:value12:value13,key2:value21,key3:value31:value32".
- * @param str delimited string represented as multiple keys having multiple
- * values.
- * @param pairsDelim key-values pairs are delimited by this delimiter.
- * @param keyValuesDelim values for a key are delimited by this delimiter.
- * @return a map of key-values with each key having a set of values.
+ * Parse relation filters.
+ * @param expr Relation filter expression
+ * @return a {@link TimelineFilterList} object.
+ *
+ * @throws Exception if any problem occurs.
*/
- static TimelineFilterList parseKeyStrValuesStr(String str, String pairsDelim,
- String keyValuesDelim) {
- return parseKeyValues(str, pairsDelim, keyValuesDelim, true, true);
+ static TimelineFilterList parseRelationFilters(String expr)
+ throws TimelineParseException {
+ return parseFilters(new TimelineParserForRelationFilters(expr,
+ TimelineParseConstants.COMMA_CHAR,
+ TimelineParseConstants.COLON_DELIMITER));
}
- /**
- * Parse a delimited string and convert it into a map of key-value pairs with
- * both the key and value interpreted as strings.
- * For instance, if pairsDelim is "," and keyValDelim is ":", then the string
- * should be represented as "key1:value1,key2:value2,key3:value3".
- * @param str delimited string represented as key-value pairs.
- * @param pairsDelim key-value pairs are delimited by this delimiter.
- * @param keyValDelim key and value are delimited by this delimiter.
- * @return a map of key-value pairs with both key and value being strings.
- */
- static TimelineFilterList parseKeyStrValueStr(String str, String pairsDelim,
- String keyValDelim) {
- return parseKeyValues(str, pairsDelim, keyValDelim, true, false);
+ private static TimelineFilterList parseFilters(TimelineParser parser)
+ throws TimelineParseException {
+ try {
+ return parser.parse();
+ } finally {
+ IOUtils.closeQuietly(parser);
+ }
}
/**
- * Parse a delimited string and convert it into a map of key-value pairs with
- * key being a string and value interpreted as any object.
- * For instance, if pairsDelim is "," and keyValDelim is ":", then the string
- * should be represented as "key1:value1,key2:value2,key3:value3".
- * @param str delimited string represented as key-value pairs.
- * @param pairsDelim key-value pairs are delimited by this delimiter.
- * @param keyValDelim key and value are delimited by this delimiter.
- * @return a map of key-value pairs with key being a string and value, any
- * object.
+ * Parses config and info filters.
+ *
+ * @param expr Expression to be parsed.
+ * @param valueAsString true, if value has to be interpreted as string, false
+ * otherwise. It is true for config filters and false for info filters.
+ * @return a {@link TimelineFilterList} object.
+ * @throws TimelineParseException if any problem occurs during parsing.
*/
- static TimelineFilterList parseKeyStrValueObj(String str, String pairsDelim,
- String keyValDelim) {
- return parseKeyValues(str, pairsDelim, keyValDelim, false, false);
+ static TimelineFilterList parseKVFilters(String expr, boolean valueAsString)
+ throws TimelineParseException {
+ return parseFilters(new TimelineParserForKVFilters(expr, valueAsString));
}
/**
@@ -245,18 +169,16 @@ final class TimelineReaderWebServicesUtils {
return fieldList;
}
- static TimelineFilterList parseMetricFilters(String str,
- String delimiter) {
- if (str == null || str.isEmpty()) {
- return null;
- }
- TimelineFilterList list = new TimelineFilterList();
- String[] strs = str.split(delimiter);
- for (String aStr : strs) {
- list.addFilter(new TimelineCompareFilter(
- TimelineCompareOp.GREATER_OR_EQUAL, aStr.trim(), 0L));
- }
- return list;
+ /**
+ * Parses metric filters.
+ *
+ * @param expr Metric filter expression to be parsed.
+ * @return a {@link TimelineFilterList} object.
+ * @throws TimelineParseException if any problem occurs during parsing.
+ */
+ static TimelineFilterList parseMetricFilters(String expr)
+ throws TimelineParseException {
+ return parseFilters(new TimelineParserForNumericFilters(expr));
}
/**
@@ -299,4 +221,16 @@ final class TimelineReaderWebServicesUtils {
static String getUserName(UserGroupInformation callerUGI) {
return ((callerUGI != null) ? callerUGI.getUserName().trim() : "");
}
+
+ /**
+ * Parses confstoretrieve and metricstoretrieve.
+ * @param str String representing confs/metrics to retrieve expression.
+ *
+ * @return a {@link TimelineFilterList} object.
+ * @throws TimelineParseException if any problem occurs during parsing.
+ */
+ static TimelineFilterList parseDataToRetrieve(String expr)
+ throws TimelineParseException {
+ return parseFilters(new TimelineParserForDataToRetrieve(expr));
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
index 81902ee..1127f4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineCompareFilter.java
@@ -29,12 +29,15 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
@Unstable
public class TimelineCompareFilter extends TimelineFilter {
- private final TimelineCompareOp compareOp;
- private final String key;
- private final Object value;
+ private TimelineCompareOp compareOp;
+ private String key;
+ private Object value;
// If comparison operator is NOT_EQUAL, this flag decides if we should return
// the entity if key does not exist.
- private final boolean keyMustExist;
+ private boolean keyMustExist = true;
+
+ public TimelineCompareFilter() {
+ }
public TimelineCompareFilter(TimelineCompareOp op, String key, Object val,
boolean keyMustExistFlag) {
@@ -65,15 +68,77 @@ public class TimelineCompareFilter extends TimelineFilter {
return key;
}
+ public void setKey(String keyToBeSet) {
+ key = keyToBeSet;
+ }
+
public Object getValue() {
return value;
}
+ public void setCompareOp(TimelineCompareOp timelineCompareOp,
+ boolean keyExistFlag) {
+ this.compareOp = timelineCompareOp;
+ if (timelineCompareOp == TimelineCompareOp.NOT_EQUAL) {
+ this.keyMustExist = keyExistFlag;
+ }
+ }
+
+ public void setValue(Object val) {
+ value = val;
+ }
+
public boolean getKeyMustExist() {
return keyMustExist;
}
@Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((compareOp == null) ? 0 : compareOp.hashCode());
+ result = prime * result + ((key == null) ? 0 : key.hashCode());
+ result = prime * result + (keyMustExist ? 1231 : 1237);
+ result = prime * result + ((value == null) ? 0 : value.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TimelineCompareFilter other = (TimelineCompareFilter) obj;
+ if (compareOp != other.compareOp) {
+ return false;
+ }
+ if (key == null) {
+ if (other.key != null) {
+ return false;
+ }
+ } else if (!key.equals(other.key)) {
+ return false;
+ }
+ if (keyMustExist != other.keyMustExist) {
+ return false;
+ }
+ if (value == null) {
+ if (other.value != null) {
+ return false;
+ }
+ } else if (!value.equals(other.value)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
public String toString() {
return String.format("%s (%s, %s:%s:%b)",
this.getClass().getSimpleName(), this.compareOp.name(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
index 36d0d7b..b4c8e25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
@@ -29,8 +29,11 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
@Unstable
public class TimelineExistsFilter extends TimelineFilter {
- private final TimelineCompareOp compareOp;
- private final String value;
+ private TimelineCompareOp compareOp;
+ private String value;
+
+ public TimelineExistsFilter() {
+ }
public TimelineExistsFilter(TimelineCompareOp op, String value) {
this.value = value;
@@ -42,14 +45,56 @@ public class TimelineExistsFilter extends TimelineFilter {
}
@Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((compareOp == null) ? 0 : compareOp.hashCode());
+ result = prime * result + ((value == null) ? 0 : value.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TimelineExistsFilter other = (TimelineExistsFilter) obj;
+ if (compareOp != other.compareOp) {
+ return false;
+ }
+ if (value == null) {
+ if (other.value != null) {
+ return false;
+ }
+ } else if (!value.equals(other.value)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
public TimelineFilterType getFilterType() {
return TimelineFilterType.EXISTS;
}
+ public void setValue(String val) {
+ value = val;
+ }
+
public String getValue() {
return value;
}
+ public void setCompareOp(TimelineCompareOp op) {
+ compareOp = op;
+ }
+
public TimelineCompareOp getCompareOp() {
return compareOp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
index 57b41a6..b4c7ad2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterList.java
@@ -98,6 +98,42 @@ public class TimelineFilterList extends TimelineFilter {
}
@Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result =
+ prime * result + ((filterList == null) ? 0 : filterList.hashCode());
+ result =
+ prime * result + ((operator == null) ? 0 : operator.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TimelineFilterList other = (TimelineFilterList) obj;
+ if (operator != other.operator) {
+ return false;
+ }
+ if (filterList == null) {
+ if (other.filterList != null) {
+ return false;
+ }
+ } else if (!filterList.equals(other.filterList)) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
public String toString() {
return String.format("TimelineFilterList %s (%d): %s",
this.operator, this.filterList.size(), this.filterList.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
index 58f0ee9..8bc8584 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
@@ -28,6 +28,9 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
@Private
@Unstable
public class TimelineKeyValueFilter extends TimelineCompareFilter {
+ public TimelineKeyValueFilter() {
+ }
+
public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val,
boolean keyMustExistFlag) {
super(op, key, val, keyMustExistFlag);
@@ -45,4 +48,14 @@ public class TimelineKeyValueFilter extends TimelineCompareFilter {
public TimelineFilterType getFilterType() {
return TimelineFilterType.KEY_VALUE;
}
+
+ public void setCompareOp(TimelineCompareOp timelineCompareOp,
+ boolean keyExistFlag) {
+ if (timelineCompareOp != TimelineCompareOp.EQUAL &&
+ timelineCompareOp != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("TimelineCompareOp for equality"
+ + " filter should be EQUAL or NOT_EQUAL");
+ }
+ super.setCompareOp(timelineCompareOp, keyExistFlag);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
index 0d34d47..fe4f6b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
@@ -31,9 +31,13 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
@Private
@Unstable
public class TimelineKeyValuesFilter extends TimelineFilter {
- private final TimelineCompareOp compareOp;
- private final String key;
- private final Set<Object> values;
+ private TimelineCompareOp compareOp;
+ private String key;
+ private Set<Object> values;
+
+ public TimelineKeyValuesFilter() {
+ }
+
public TimelineKeyValuesFilter(TimelineCompareOp op, String key,
Set<Object> values) {
if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
@@ -58,6 +62,15 @@ public class TimelineKeyValuesFilter extends TimelineFilter {
return values;
}
+ public void setKeyAndValues(String keyForValues, Set<Object> vals) {
+ key = keyForValues;
+ values = vals;
+ }
+
+ public void setCompareOp(TimelineCompareOp op) {
+ compareOp = op;
+ }
+
public TimelineCompareOp getCompareOp() {
return compareOp;
}
@@ -68,4 +81,46 @@ public class TimelineKeyValuesFilter extends TimelineFilter {
this.getClass().getSimpleName(), this.compareOp.name(),
this.key, (values == null) ? "" : values.toString());
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((compareOp == null) ? 0 : compareOp.hashCode());
+ result = prime * result + ((key == null) ? 0 : key.hashCode());
+ result = prime * result + ((values == null) ? 0 : values.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TimelineKeyValuesFilter other = (TimelineKeyValuesFilter) obj;
+ if (compareOp != other.compareOp) {
+ return false;
+ }
+ if (key == null) {
+ if (other.key != null) {
+ return false;
+ }
+ } else if (!key.equals(other.key)) {
+ return false;
+ }
+ if (values == null) {
+ if (other.values != null) {
+ return false;
+ }
+ } else if (!values.equals(other.values)) {
+ return false;
+ }
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
index f36e593..bbdc960 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelinePrefixFilter.java
@@ -32,6 +32,9 @@ public class TimelinePrefixFilter extends TimelineFilter {
private TimelineCompareOp compareOp;
private String prefix;
+ public TimelinePrefixFilter() {
+ }
+
public TimelinePrefixFilter(TimelineCompareOp op, String prefix) {
this.prefix = prefix;
if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
@@ -59,4 +62,38 @@ public class TimelinePrefixFilter extends TimelineFilter {
return String.format("%s (%s %s)",
this.getClass().getSimpleName(), this.compareOp.name(), this.prefix);
}
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((compareOp == null) ? 0 : compareOp.hashCode());
+ result = prime * result + ((prefix == null) ? 0 : prefix.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TimelinePrefixFilter other = (TimelinePrefixFilter) obj;
+ if (compareOp != other.compareOp) {
+ return false;
+ }
+ if (prefix == null) {
+ if (other.prefix != null) {
+ return false;
+ }
+ } else if (!prefix.equals(other.prefix)){
+ return false;
+ }
+ return true;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 2bd2830..6c3f3e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -437,7 +437,7 @@ public class TestTimelineReaderWebServices {
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/apps/app1/entities/app?" +
- "conffilters=config_1:123,config_3:abc");
+ "conffilters=config_1%20eq%20123%20AND%20config_3%20eq%20abc");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -457,7 +457,7 @@ public class TestTimelineReaderWebServices {
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/apps/app1/entities/app?" +
- "infofilters=info2:3.5");
+ "infofilters=info2%20eq%203.5");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -477,7 +477,7 @@ public class TestTimelineReaderWebServices {
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/apps/app1/entities/app?" +
- "metricfilters=metric3");
+ "metricfilters=metric3%20ge%200");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -519,10 +519,10 @@ public class TestTimelineReaderWebServices {
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/apps/app1/entities/app?" +
- "metricfilters=metric7&isrelatedto=type1:tid1_1;tid1_2,type2:tid2_1" +
- "%60&relatesto=flow:flow1&eventfilters=event_2,event_4&infofilters=" +
- "info2:3.5&createdtimestart=1425016502030&createdtimeend=" +
- "1425016502060");
+ "metricfilters=metric7%20ge%200&isrelatedto=type1:tid1_1:tid1_2,"+
+ "type2:tid2_1%60&relatesto=flow:flow1&eventfilters=event_2,event_4" +
+ "&infofilters=info2%20eq%203.5&createdtimestart=1425016502030&" +
+ "createdtimeend=1425016502060");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 009b488..ca80ed5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -60,6 +60,8 @@ import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Sets;
import com.sun.jersey.api.client.Client;
import com.sun.jersey.api.client.ClientResponse;
import com.sun.jersey.api.client.ClientResponse.Status;
@@ -109,18 +111,20 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Set<TimelineMetric> metrics = new HashSet<>();
TimelineMetric m1 = new TimelineMetric();
m1.setId("MAP_SLOT_MILLIS");
- Map<Long, Number> metricValues = new HashMap<Long, Number>();
- metricValues.put(ts - 100000, 2);
- metricValues.put(ts - 80000, 40);
+ Map<Long, Number> metricValues =
+ ImmutableMap.of(ts - 100000, (Number)2, ts - 80000, 40);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ m1 = new TimelineMetric();
+ m1.setId("MAP1_SLOT_MILLIS");
+ metricValues = ImmutableMap.of(ts - 100000, (Number)2, ts - 80000, 40);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
-
m1 = new TimelineMetric();
m1.setId("HDFS_BYTES_READ");
- metricValues = new HashMap<Long, Number>();
- metricValues.put(ts - 100000, 31);
- metricValues.put(ts - 80000, 57);
+ metricValues = ImmutableMap.of(ts - 100000, (Number)31, ts - 80000, 57);
m1.setType(Type.TIME_SERIES);
m1.setValues(metricValues);
metrics.add(m1);
@@ -198,6 +202,14 @@ public class TestTimelineReaderWebServicesHBaseStorage {
event4.setTimestamp(cTime);
event4.addInfo("foo_event", "test");
entity4.addEvent(event4);
+ metrics.clear();
+ m2 = new TimelineMetric();
+ m2.setId("MAP_SLOT_MILLIS");
+ metricValues = ImmutableMap.of(ts - 100000, (Number)5L, ts - 80000, 101L);
+ m2.setType(Type.TIME_SERIES);
+ m2.setValues(metricValues);
+ metrics.add(m2);
+ entity4.addMetrics(metrics);
te4.addEntity(entity4);
TimelineEntities te5 = new TimelineEntities();
@@ -205,11 +217,116 @@ public class TestTimelineReaderWebServicesHBaseStorage {
entity5.setId("entity1");
entity5.setType("type1");
entity5.setCreatedTime(1425016501034L);
+ // add some config entries
+ entity5.addConfigs(ImmutableMap.of("config_param1", "value1",
+ "config_param2", "value2", "cfg_param1", "value3"));
+ entity5.addInfo(ImmutableMap.of("info1", (Object)"cluster1",
+ "info2", 2.0, "info3", 35000, "info4", 36000));
+ metrics = new HashSet<>();
+ m1 = new TimelineMetric();
+ m1.setId("MAP_SLOT_MILLIS");
+ metricValues = ImmutableMap.of(ts - 100000, (Number)2, ts - 80000, 40);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ m1 = new TimelineMetric();
+ m1.setId("HDFS_BYTES_READ");
+ metricValues = ImmutableMap.of(ts - 100000, (Number)31, ts - 80000, 57);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ entity5.addMetrics(metrics);
+ TimelineEvent event51 = new TimelineEvent();
+ event51.setId("event1");
+ event51.setTimestamp(cTime);
+ entity5.addEvent(event51);
+ TimelineEvent event52 = new TimelineEvent();
+ event52.setId("event2");
+ event52.setTimestamp(cTime);
+ entity5.addEvent(event52);
+ TimelineEvent event53 = new TimelineEvent();
+ event53.setId("event3");
+ event53.setTimestamp(cTime);
+ entity5.addEvent(event53);
+ TimelineEvent event54 = new TimelineEvent();
+ event54.setId("event4");
+ event54.setTimestamp(cTime);
+ entity5.addEvent(event54);
+ Map<String, Set<String>> isRelatedTo1 = new HashMap<String, Set<String>>();
+ isRelatedTo1.put("type2",
+ Sets.newHashSet("entity21","entity22","entity23","entity24"));
+ isRelatedTo1.put("type4", Sets.newHashSet("entity41","entity42"));
+ isRelatedTo1.put("type1", Sets.newHashSet("entity14","entity15"));
+ isRelatedTo1.put("type3",
+ Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
+ entity5.addIsRelatedToEntities(isRelatedTo1);
+ Map<String, Set<String>> relatesTo1 = new HashMap<String, Set<String>>();
+ relatesTo1.put("type2",
+ Sets.newHashSet("entity21","entity22","entity23","entity24"));
+ relatesTo1.put("type4", Sets.newHashSet("entity41","entity42"));
+ relatesTo1.put("type1", Sets.newHashSet("entity14","entity15"));
+ relatesTo1.put("type3",
+ Sets.newHashSet("entity31", "entity35", "entity32", "entity33"));
+ entity5.addRelatesToEntities(relatesTo1);
te5.addEntity(entity5);
+
TimelineEntity entity6 = new TimelineEntity();
entity6.setId("entity2");
entity6.setType("type1");
entity6.setCreatedTime(1425016501034L);
+ entity6.addConfigs(ImmutableMap.of("cfg_param3", "value1",
+ "configuration_param2", "value2", "config_param1", "value3"));
+ entity6.addInfo(ImmutableMap.of("info1", (Object)"cluster2",
+ "info2", 2.0, "info4", 35000));
+ metrics = new HashSet<>();
+ m1 = new TimelineMetric();
+ m1.setId("MAP1_SLOT_MILLIS");
+ metricValues = ImmutableMap.of(ts - 100000, (Number)12, ts - 80000, 140);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ m1 = new TimelineMetric();
+ m1.setId("HDFS_BYTES_READ");
+ metricValues = ImmutableMap.of(ts - 100000, (Number)78, ts - 80000, 157);
+ m1.setType(Type.TIME_SERIES);
+ m1.setValues(metricValues);
+ metrics.add(m1);
+ m1 = new TimelineMetric();
+ m1.setId("MAP11_SLOT_MILLIS");
+ m1.setType(Type.SINGLE_VALUE);
+ m1.addValue(ts - 100000, 122);
+ metrics.add(m1);
+ entity6.addMetrics(metrics);
+ TimelineEvent event61 = new TimelineEvent();
+ event61.setId("event1");
+ event61.setTimestamp(cTime);
+ entity6.addEvent(event61);
+ TimelineEvent event62 = new TimelineEvent();
+ event62.setId("event5");
+ event62.setTimestamp(cTime);
+ entity6.addEvent(event62);
+ TimelineEvent event63 = new TimelineEvent();
+ event63.setId("event3");
+ event63.setTimestamp(cTime);
+ entity6.addEvent(event63);
+ TimelineEvent event64 = new TimelineEvent();
+ event64.setId("event6");
+ event64.setTimestamp(cTime);
+ entity6.addEvent(event64);
+ Map<String, Set<String>> isRelatedTo2 = new HashMap<String, Set<String>>();
+ isRelatedTo2.put("type2",
+ Sets.newHashSet("entity21","entity22","entity23","entity24"));
+ isRelatedTo2.put("type5", Sets.newHashSet("entity51","entity52"));
+ isRelatedTo2.put("type6", Sets.newHashSet("entity61","entity66"));
+ isRelatedTo2.put("type3", Sets.newHashSet("entity31"));
+ entity6.addIsRelatedToEntities(isRelatedTo2);
+ Map<String, Set<String>> relatesTo2 = new HashMap<String, Set<String>>();
+ relatesTo2.put("type2",
+ Sets.newHashSet("entity21","entity22","entity23","entity24"));
+ relatesTo2.put("type5", Sets.newHashSet("entity51","entity52"));
+ relatesTo2.put("type6", Sets.newHashSet("entity61","entity66"));
+ relatesTo2.put("type3", Sets.newHashSet("entity31"));
+ entity6.addRelatesToEntities(relatesTo2);
te5.addEntity(entity6);
HBaseTimelineWriterImpl hbi = null;
@@ -357,13 +474,15 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
assertNotNull(entity);
assertEquals("user1@flow_name/1002345678919", entity.getId());
- assertEquals(2, entity.getMetrics().size());
+ assertEquals(3, entity.getMetrics().size());
TimelineMetric m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
"HDFS_BYTES_READ", ts - 80000, 57L);
TimelineMetric m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
"MAP_SLOT_MILLIS", ts - 80000, 141L);
+ TimelineMetric m3 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
+ "MAP1_SLOT_MILLIS", ts - 80000, 40L);
for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m1, m2));
+ assertTrue(verifyMetrics(metric, m1, m2, m3));
}
// Query without specifying cluster ID.
@@ -373,20 +492,21 @@ public class TestTimelineReaderWebServicesHBaseStorage {
entity = resp.getEntity(FlowRunEntity.class);
assertNotNull(entity);
assertEquals("user1@flow_name/1002345678919", entity.getId());
- assertEquals(2, entity.getMetrics().size());
+ assertEquals(3, entity.getMetrics().size());
m1 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
"HDFS_BYTES_READ", ts - 80000, 57L);
m2 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
"MAP_SLOT_MILLIS", ts - 80000, 141L);
+ m3 = newMetric(TimelineMetric.Type.SINGLE_VALUE,
+ "MAP1_SLOT_MILLIS", ts - 80000, 40L);
for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m1, m2));
+ assertTrue(verifyMetrics(metric, m1, m2, m3));
}
} finally {
client.destroy();
}
}
-
@Test
public void testGetFlowRuns() throws Exception {
Client client = createClient();
@@ -489,12 +609,57 @@ public class TestTimelineReaderWebServicesHBaseStorage {
((entity.getId().equals("user1@flow_name/1002345678919")) &&
(entity.getRunId() == 1002345678919L) &&
(entity.getStartTime() == 1425016501000L) &&
- (entity.getMetrics().size() == 2)) ||
+ (entity.getMetrics().size() == 3)) ||
((entity.getId().equals("user1@flow_name/1002345678920")) &&
(entity.getRunId() == 1002345678920L) &&
(entity.getStartTime() == 1425016501034L) &&
- (entity.getMetrics().size() == 0)));
+ (entity.getMetrics().size() == 1)));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+
+ @Test
+ public void testGetFlowRunsMetricsToRetrieve() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" +
+ "metricstoretrieve=MAP_,HDFS_");
+ ClientResponse resp = getResponse(client, uri);
+ Set<FlowRunEntity> entities =
+ resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ int metricCnt = 0;
+ for (FlowRunEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP_") ||
+ metric.getId().startsWith("HDFS_"));
+ }
+ }
+ assertEquals(3, metricCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" +
+ "metricstoretrieve=!(MAP_,HDFS_)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ metricCnt = 0;
+ for (FlowRunEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP1_"));
+ }
}
+ assertEquals(1, metricCnt);
} finally {
client.destroy();
}
@@ -858,15 +1023,18 @@ public class TestTimelineReaderWebServicesHBaseStorage {
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertNotNull(entity);
assertEquals("application_1111111111_1111", entity.getId());
- assertEquals(2, entity.getMetrics().size());
+ assertEquals(3, entity.getMetrics().size());
TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES,
"HDFS_BYTES_READ", ts - 100000, 31L);
m1.addValue(ts - 80000, 57L);
TimelineMetric m2 = newMetric(TimelineMetric.Type.TIME_SERIES,
"MAP_SLOT_MILLIS", ts - 100000, 2L);
m2.addValue(ts - 80000, 40L);
+ TimelineMetric m3 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "MAP1_SLOT_MILLIS", ts - 100000, 2L);
+ m3.addValue(ts - 80000, 40L);
for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m1, m2));
+ assertTrue(verifyMetrics(metric, m1, m2, m3));
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
@@ -877,11 +1045,11 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertNotNull(entity);
assertEquals("application_1111111111_2222", entity.getId());
assertEquals(1, entity.getMetrics().size());
- TimelineMetric m3 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ TimelineMetric m4 = newMetric(TimelineMetric.Type.TIME_SERIES,
"MAP_SLOT_MILLIS", ts - 100000, 5L);
- m2.addValue(ts - 80000, 101L);
+ m4.addValue(ts - 80000, 101L);
for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m3));
+ assertTrue(verifyMetrics(metric, m4));
}
} finally {
client.destroy();
@@ -899,15 +1067,18 @@ public class TestTimelineReaderWebServicesHBaseStorage {
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertNotNull(entity);
assertEquals("application_1111111111_1111", entity.getId());
- assertEquals(2, entity.getMetrics().size());
+ assertEquals(3, entity.getMetrics().size());
TimelineMetric m1 = newMetric(TimelineMetric.Type.TIME_SERIES,
"HDFS_BYTES_READ", ts - 100000, 31L);
m1.addValue(ts - 80000, 57L);
TimelineMetric m2 = newMetric(TimelineMetric.Type.TIME_SERIES,
"MAP_SLOT_MILLIS", ts - 100000, 2L);
m2.addValue(ts - 80000, 40L);
+ TimelineMetric m3 = newMetric(TimelineMetric.Type.TIME_SERIES,
+ "MAP1_SLOT_MILLIS", ts - 100000, 2L);
+ m3.addValue(ts - 80000, 40L);
for (TimelineMetric metric : entity.getMetrics()) {
- assertTrue(verifyMetrics(metric, m1, m2));
+ assertTrue(verifyMetrics(metric, m1, m2, m3));
}
} finally {
client.destroy();
@@ -952,6 +1123,683 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
}
+ /**
+ * Tests if specific configs and metrics are retrieve for getEntities call.
+ */
+ @Test
+ public void testGetEntitiesDataToRetrieve() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?confstoretrieve=cfg_");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ int cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ for (String configKey : entity.getConfigs().keySet()) {
+ assertTrue(configKey.startsWith("cfg_"));
+ }
+ }
+ assertEquals(2, cfgCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?confstoretrieve=cfg_,config_");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ for (String configKey : entity.getConfigs().keySet()) {
+ assertTrue(configKey.startsWith("cfg_") ||
+ configKey.startsWith("config_"));
+ }
+ }
+ assertEquals(5, cfgCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?confstoretrieve=!(cfg_,config_)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ for (String configKey : entity.getConfigs().keySet()) {
+ assertTrue(configKey.startsWith("configuration_"));
+ }
+ }
+ assertEquals(1, cfgCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricstoretrieve=MAP_");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ int metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP_"));
+ }
+ }
+ assertEquals(1, metricCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricstoretrieve=MAP1_,HDFS_");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP1_") ||
+ metric.getId().startsWith("HDFS_"));
+ }
+ }
+ assertEquals(3, metricCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricstoretrieve=!(MAP1_,HDFS_)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP_") ||
+ metric.getId().startsWith("MAP11_"));
+ }
+ }
+ assertEquals(2, metricCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricstoretrieve=!(MAP1_,HDFS_");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesConfigFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=config_param1%20eq%20value1%20OR%20" +
+ "config_param1%20eq%20value3");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=config_param1%20eq%20value1%20AND" +
+ "%20configuration_param2%20eq%20value2");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+
+ // conffilters=(config_param1 eq value1 AND configuration_param2 eq
+ // value2) OR (config_param1 eq value3 AND cfg_param3 eq value1)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=(config_param1%20eq%20value1%20AND" +
+ "%20configuration_param2%20eq%20value2)%20OR%20(config_param1%20eq" +
+ "%20value3%20AND%20cfg_param3%20eq%20value1)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ int cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ assertEquals(0, cfgCnt);
+
+ // conffilters=(config_param1 eq value1 AND configuration_param2 eq
+ // value2) OR (config_param1 eq value3 AND cfg_param3 eq value1)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=(config_param1%20eq%20value1%20AND" +
+ "%20configuration_param2%20eq%20value2)%20OR%20(config_param1%20eq" +
+ "%20value3%20AND%20cfg_param3%20eq%20value1)&fields=CONFIGS");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ assertEquals(3, cfgCnt);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=(config_param1%20eq%20value1%20AND" +
+ "%20configuration_param2%20eq%20value2)%20OR%20(config_param1%20eq" +
+ "%20value3%20AND%20cfg_param3%20eq%20value1)&confstoretrieve=cfg_," +
+ "configuration_");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ cfgCnt = 0;
+ for (TimelineEntity entity : entities) {
+ cfgCnt += entity.getConfigs().size();
+ assertTrue(entity.getId().equals("entity2"));
+ for (String configKey : entity.getConfigs().keySet()) {
+ assertTrue(configKey.startsWith("cfg_") ||
+ configKey.startsWith("configuration_"));
+ }
+ }
+ assertEquals(2, cfgCnt);
+
+ // Test for behavior when compare op is ne(not equals) vs ene
+ // (exists and not equals). configuration_param2 does not exist for
+ // entity1. For ne, both entity1 and entity2 will be returned. For ene,
+ // only entity2 will be returned as we are checking for existence too.
+ // conffilters=configuration_param2 ne value3
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=configuration_param2%20ne%20value3");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+ // conffilters=configuration_param2 ene value3
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?conffilters=configuration_param2%20ene%20value3");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesInfoFilters() throws Exception {
+ Client client = createClient();
+ try {
+ // infofilters=info1 eq cluster1 OR info1 eq cluster2
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=info1%20eq%20cluster1%20OR%20info1%20eq" +
+ "%20cluster2");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ // infofilters=info1 eq cluster1 AND info4 eq 35000
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=info1%20eq%20cluster1%20AND%20info4%20" +
+ "eq%2035000");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+
+ // infofilters=info4 eq 35000 OR info4 eq 36000
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=info4%20eq%2035000%20OR%20info4%20eq" +
+ "%2036000");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ // infofilters=(info1 eq cluster1 AND info4 eq 35000) OR
+ // (info1 eq cluster2 AND info2 eq 2.0)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=(info1%20eq%20cluster1%20AND%20info4%20" +
+ "eq%2035000)%20OR%20(info1%20eq%20cluster2%20AND%20info2%20eq%202.0)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ int infoCnt = 0;
+ for (TimelineEntity entity : entities) {
+ infoCnt += entity.getInfo().size();
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ // Includes UID in info field even if fields not specified as INFO.
+ assertEquals(1, infoCnt);
+
+ // infofilters=(info1 eq cluster1 AND info4 eq 35000) OR
+ // (info1 eq cluster2 AND info2 eq 2.0)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=(info1%20eq%20cluster1%20AND%20info4%20" +
+ "eq%2035000)%20OR%20(info1%20eq%20cluster2%20AND%20info2%20eq%20" +
+ "2.0)&fields=INFO");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ infoCnt = 0;
+ for (TimelineEntity entity : entities) {
+ infoCnt += entity.getInfo().size();
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ // Includes UID in info field.
+ assertEquals(4, infoCnt);
+
+ // Test for behavior when compare op is ne(not equals) vs ene
+ // (exists and not equals). info3 does not exist for entity2. For ne,
+ // both entity1 and entity2 will be returned. For ene, only entity2 will
+ // be returned as we are checking for existence too.
+ // infofilters=info3 ne 39000
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=info3%20ne%2039000");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+ // infofilters=info3 ene 39000
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?infofilters=info3%20ene%2039000");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1"));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesMetricFilters() throws Exception {
+ Client client = createClient();
+ try {
+ // metricfilters=HDFS_BYTES_READ lt 60 OR HDFS_BYTES_READ eq 157
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=HDFS_BYTES_READ%20lt%2060%20OR%20" +
+ "HDFS_BYTES_READ%20eq%20157");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ // metricfilters=HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=HDFS_BYTES_READ%20lt%2060%20AND%20" +
+ "MAP_SLOT_MILLIS%20gt%2040");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+
+ // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR
+ // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" +
+ "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" +
+ "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ int metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ assertEquals(0, metricCnt);
+
+ // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR
+ // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" +
+ "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" +
+ "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)&fields=METRICS");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ assertEquals(3, metricCnt);
+
+ // metricfilters=(HDFS_BYTES_READ lt 60 AND MAP_SLOT_MILLIS gt 40) OR
+ // (MAP1_SLOT_MILLIS ge 140 AND MAP11_SLOT_MILLIS le 122)
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=(HDFS_BYTES_READ%20lt%2060%20AND%20" +
+ "MAP_SLOT_MILLIS%20gt%2040)%20OR%20(MAP1_SLOT_MILLIS%20ge" +
+ "%20140%20AND%20MAP11_SLOT_MILLIS%20le%20122)&metricstoretrieve=" +
+ "!(HDFS)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ assertTrue(entity.getId().equals("entity2"));
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP1"));
+ }
+ }
+ assertEquals(2, metricCnt);
+
+ // Test for behavior when compare op is ne(not equals) vs ene
+ // (exists and not equals). MAP11_SLOT_MILLIS does not exist for
+ // entity1. For ne, both entity1 and entity2 will be returned. For ene,
+ // only entity2 will be returned as we are checking for existence too.
+ // metricfilters=MAP11_SLOT_MILLIS ne 100
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=MAP11_SLOT_MILLIS%20ne%20100");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+ // metricfilters=MAP11_SLOT_MILLIS ene 100
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?metricfilters=MAP11_SLOT_MILLIS%20ene%20100");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity2"));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesEventFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?eventfilters=event1,event3");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?eventfilters=!(event1,event3)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+
+ // eventfilters=!(event1,event3) OR event5,event6
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?eventfilters=!(event1,event3)%20OR%20event5,event6");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity2"));
+ }
+
+ // eventfilters=(!(event1,event3) OR event5,event6) OR
+ // (event1,event2 AND (event3,event4))
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?eventfilters=(!(event1,event3)%20OR%20event5," +
+ "event6)%20OR%20(event1,event2%20AND%20(event3,event4))");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testGetEntitiesRelationFilters() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1?isrelatedto=type3:entity31,type2:entity21:entity22");
+ ClientResponse resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "clusters/cluster1/apps/application_1111111111_1111/entities/type1" +
+ "?isrelatedto=!(type3:entity31,type2:entity21:entity22)");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(0, entities.size());
+
+ // isrelatedto=!(type3:entity31,type2:entity21:entity22)OR type5:entity51,
+ // type6:entity61:entity66
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "clusters/cluster1/apps/application_1111111111_1111/entities/type1" +
+ "?isrelatedto=!(type3:entity31,type2:entity21:entity22)%20OR%20" +
+ "type5:entity51,type6:entity61:entity66");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity2"));
+ }
+
+ // isrelatedto=(!(type3:entity31,type2:entity21:entity22)OR type5:
+ // entity51,type6:entity61:entity66) OR (type1:entity14,type2:entity21:
+ // entity22 AND (type3:entity32:entity35,type4:entity42))
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "clusters/cluster1/apps/application_1111111111_1111/entities/type1" +
+ "?isrelatedto=(!(type3:entity31,type2:entity21:entity22)%20OR%20" +
+ "type5:entity51,type6:entity61:entity66)%20OR%20(type1:entity14," +
+ "type2:entity21:entity22%20AND%20(type3:entity32:entity35,"+
+ "type4:entity42))");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+
+ // relatesto=!(type3:entity31,type2:entity21:entity22)OR type5:entity51,
+ // type6:entity61:entity66
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "clusters/cluster1/apps/application_1111111111_1111/entities/type1" +
+ "?relatesto=!%20(type3:entity31,type2:entity21:entity22%20)%20OR%20" +
+ "type5:entity51,type6:entity61:entity66");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity2"));
+ }
+
+ // relatesto=(!(type3:entity31,type2:entity21:entity22)OR type5:entity51,
+ // type6:entity61:entity66) OR (type1:entity14,type2:entity21:entity22 AND
+ // (type3:entity32:entity35 , type4:entity42))
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "clusters/cluster1/apps/application_1111111111_1111/entities/type1" +
+ "?relatesto=(!(%20type3:entity31,type2:entity21:entity22)%20OR%20" +
+ "type5:entity51,type6:entity61:entity66%20)%20OR%20(type1:entity14," +
+ "type2:entity21:entity22%20AND%20(type3:entity32:entity35%20,%20"+
+ "type4:entity42))");
+ resp = getResponse(client, uri);
+ entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ assertTrue(entity.getId().equals("entity1") ||
+ entity.getId().equals("entity2"));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
+ /**
+ * Tests if specific configs and metrics are retrieve for getEntity call.
+ */
+ @Test
+ public void testGetEntityDataToRetrieve() throws Exception {
+ Client client = createClient();
+ try {
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity2?confstoretrieve=cfg_,configuration_");
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("entity2", entity.getId());
+ assertEquals("type1", entity.getType());
+ assertEquals(2, entity.getConfigs().size());
+ for (String configKey : entity.getConfigs().keySet()) {
+ assertTrue(configKey.startsWith("configuration_") ||
+ configKey.startsWith("cfg_"));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity2?confstoretrieve=!(cfg_,configuration_)");
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("entity2", entity.getId());
+ assertEquals("type1", entity.getType());
+ assertEquals(1, entity.getConfigs().size());
+ for (String configKey : entity.getConfigs().keySet()) {
+ assertTrue(configKey.startsWith("config_"));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity2?metricstoretrieve=MAP1_,HDFS_");
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("entity2", entity.getId());
+ assertEquals("type1", entity.getType());
+ assertEquals(2, entity.getMetrics().size());
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP1_") ||
+ metric.getId().startsWith("HDFS_"));
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity2?metricstoretrieve=!(MAP1_,HDFS_)");
+ resp = getResponse(client, uri);
+ entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ assertEquals("entity2", entity.getId());
+ assertEquals("type1", entity.getType());
+ assertEquals(1, entity.getMetrics().size());
+ for (TimelineMetric metric : entity.getMetrics()) {
+ assertTrue(metric.getId().startsWith("MAP11_"));
+ }
+ } finally {
+ client.destroy();
+ }
+ }
+
@Test
public void testGetFlowRunApps() throws Exception {
Client client = createClient();
@@ -967,7 +1815,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
for (TimelineEntity entity : entities) {
assertTrue("Unexpected app in result",
(entity.getId().equals("application_1111111111_1111") &&
- entity.getMetrics().size() == 2) ||
+ entity.getMetrics().size() == 3) ||
(entity.getId().equals("application_1111111111_2222") &&
entity.getMetrics().size() == 1));
}
@@ -1007,11 +1855,11 @@ public class TestTimelineReaderWebServicesHBaseStorage {
for (TimelineEntity entity : entities) {
assertTrue("Unexpected app in result",
(entity.getId().equals("application_1111111111_1111") &&
- entity.getMetrics().size() == 2) ||
+ entity.getMetrics().size() == 3) ||
(entity.getId().equals("application_1111111111_2222") &&
entity.getMetrics().size() == 1) ||
(entity.getId().equals("application_1111111111_2224") &&
- entity.getMetrics().size() == 0));
+ entity.getMetrics().size() == 1));
}
// Query without specifying cluster ID.
@@ -1051,7 +1899,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" +
- "metricfilters=HDFS_BYTES_READ");
+ "metricfilters=HDFS_BYTES_READ%20ge%200");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
@@ -1061,7 +1909,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
"timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" +
- "conffilters=cfg1:value1");
+ "conffilters=cfg1%20eq%20value1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: YARN-4224. Support fetching
entities by UID and change the REST interface to conform to current REST
APIs' in YARN. (Varun Saxena via gtcarrera9)
Posted by gt...@apache.org.
YARN-4224. Support fetching entities by UID and change the REST
interface to conform to current REST APIs' in YARN. (Varun Saxena via
gtcarrera9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/97d5cf32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/97d5cf32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/97d5cf32
Branch: refs/heads/YARN-2928
Commit: 97d5cf323128e9d6e3f832de891b90e1434fee7d
Parents: fd369a5
Author: Li Lu <gt...@apache.org>
Authored: Wed Jan 27 14:04:09 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:24:15 2016 -0700
----------------------------------------------------------------------
.../records/timelineservice/TimelineEntity.java | 13 +
.../server/timelineservice/TimelineContext.java | 146 ++
.../collector/TimelineCollectorContext.java | 86 +-
.../server/timelineservice/package-info.java | 28 +
.../reader/TimelineReaderContext.java | 88 ++
.../reader/TimelineReaderManager.java | 82 +-
.../reader/TimelineReaderUtils.java | 171 ++
.../reader/TimelineReaderWebServices.java | 1469 +++++++++++++++---
.../reader/TimelineReaderWebServicesUtils.java | 222 +++
.../reader/TimelineUIDConverter.java | 245 +++
.../timelineservice/storage/TimelineReader.java | 16 +-
.../storage/reader/GenericEntityReader.java | 7 +-
.../reader/TestTimelineReaderUtils.java | 55 +
.../reader/TestTimelineReaderWebServices.java | 83 +-
...stTimelineReaderWebServicesHBaseStorage.java | 348 ++++-
.../reader/TestTimelineUIDConverter.java | 97 ++
...TestPhoenixOfflineAggregationWriterImpl.java | 2 +-
17 files changed, 2781 insertions(+), 377 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index dcf2473..a661f7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -504,6 +504,19 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
}
+ /**
+ * Set UID in info which will be then used for query by UI.
+ * @param uidKey key for UID in info.
+ * @param uId UID to be set for the key.
+ */
+ public void setUID(String uidKey, String uId) {
+ if (real == null) {
+ info.put(uidKey, uId);
+ } else {
+ real.addInfo(uidKey, uId);
+ }
+ }
+
public boolean isValid() {
return (getId() != null && getType() != null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
new file mode 100644
index 0000000..694b709
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/TimelineContext.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice;
+
+/**
+ * Encapsulates timeline context information.
+ */
+public class TimelineContext {
+
+ private String clusterId;
+ private String userId;
+ private String flowName;
+ private Long flowRunId;
+ private String appId;
+
+ public TimelineContext() {
+ this(null, null, null, 0L, null);
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((appId == null) ? 0 : appId.hashCode());
+ result = prime * result + ((clusterId == null) ? 0 : clusterId.hashCode());
+ result = prime * result + ((flowName == null) ? 0 : flowName.hashCode());
+ result = prime * result + ((flowRunId == null) ? 0 : flowRunId.hashCode());
+ result = prime * result + ((userId == null) ? 0 : userId.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ TimelineContext other = (TimelineContext) obj;
+ if (appId == null) {
+ if (other.appId != null) {
+ return false;
+ }
+ } else if (!appId.equals(other.appId)) {
+ return false;
+ }
+ if (clusterId == null) {
+ if (other.clusterId != null) {
+ return false;
+ }
+ } else if (!clusterId.equals(other.clusterId)) {
+ return false;
+ }
+ if (flowName == null) {
+ if (other.flowName != null) {
+ return false;
+ }
+ } else if (!flowName.equals(other.flowName)) {
+ return false;
+ }
+ if (flowRunId == null) {
+ if (other.flowRunId != null) {
+ return false;
+ }
+ } else if (!flowRunId.equals(other.flowRunId)) {
+ return false;
+ }
+ if (userId == null) {
+ if (other.userId != null) {
+ return false;
+ }
+ } else if (!userId.equals(other.userId)) {
+ return false;
+ }
+ return true;
+ }
+
+ public TimelineContext(String clusterId, String userId, String flowName,
+ Long flowRunId, String appId) {
+ this.clusterId = clusterId;
+ this.userId = userId;
+ this.flowName = flowName;
+ this.flowRunId = flowRunId;
+ this.appId = appId;
+ }
+
+ public String getClusterId() {
+ return clusterId;
+ }
+
+ public void setClusterId(String cluster) {
+ this.clusterId = cluster;
+ }
+
+ public String getUserId() {
+ return userId;
+ }
+
+ public void setUserId(String user) {
+ this.userId = user;
+ }
+
+ public String getFlowName() {
+ return flowName;
+ }
+
+ public void setFlowName(String flow) {
+ this.flowName = flow;
+ }
+
+ public Long getFlowRunId() {
+ return flowRunId;
+ }
+
+ public void setFlowRunId(long runId) {
+ this.flowRunId = runId;
+ }
+
+ public String getAppId() {
+ return appId;
+ }
+
+ public void setAppId(String app) {
+ this.appId = app;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
index 6cc477f..58d68df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
@@ -18,74 +18,58 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
-public class TimelineCollectorContext {
+import org.apache.hadoop.yarn.server.timelineservice.TimelineContext;
+
+/**
+ * Encapsulates context information required by collector during a put.
+ */
+public class TimelineCollectorContext extends TimelineContext {
- private String clusterId;
- private String userId;
- private String flowName;
private String flowVersion;
- private long flowRunId;
- private String appId;
public TimelineCollectorContext() {
this(null, null, null, null, 0L, null);
}
public TimelineCollectorContext(String clusterId, String userId,
- String flowName, String flowVersion, long flowRunId, String appId) {
- this.clusterId = clusterId;
- this.userId = userId;
- this.flowName = flowName;
+ String flowName, String flowVersion, Long flowRunId, String appId) {
+ super(clusterId, userId, flowName, flowRunId, appId);
this.flowVersion = flowVersion;
- this.flowRunId = flowRunId;
- this.appId = appId;
- }
-
- public String getClusterId() {
- return clusterId;
- }
-
- public void setClusterId(String clusterId) {
- this.clusterId = clusterId;
- }
-
- public String getUserId() {
- return userId;
- }
-
- public void setUserId(String userId) {
- this.userId = userId;
}
- public String getFlowName() {
- return flowName;
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result =
+ prime * result + ((flowVersion == null) ? 0 : flowVersion.hashCode());
+ return result + super.hashCode();
}
- public void setFlowName(String flowName) {
- this.flowName = flowName;
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!super.equals(obj)) {
+ return false;
+ }
+ TimelineCollectorContext other = (TimelineCollectorContext) obj;
+ if (flowVersion == null) {
+ if (other.flowVersion != null) {
+ return false;
+ }
+ } else if (!flowVersion.equals(other.flowVersion)) {
+ return false;
+ }
+ return true;
}
public String getFlowVersion() {
return flowVersion;
}
- public void setFlowVersion(String flowVersion) {
- this.flowVersion = flowVersion;
- }
-
- public long getFlowRunId() {
- return flowRunId;
- }
-
- public void setFlowRunId(long flowRunId) {
- this.flowRunId = flowRunId;
- }
-
- public String getAppId() {
- return appId;
- }
-
- public void setAppId(String appId) {
- this.appId = appId;
+ public void setFlowVersion(String version) {
+ this.flowVersion = version;
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/package-info.java
new file mode 100644
index 0000000..58e23f0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.server.timelineservice contains classes to be used
+ * across timeline reader and collector.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
new file mode 100644
index 0000000..d0ad54a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import org.apache.hadoop.yarn.server.timelineservice.TimelineContext;
+
+/**
+ * Encapsulates fields necessary to make a query in timeline reader.
+ */
+public class TimelineReaderContext extends TimelineContext {
+
+ private String entityType;
+ private String entityId;
+ public TimelineReaderContext(String clusterId, String userId, String flowName,
+ Long flowRunId, String appId, String entityType, String entityId) {
+ super(clusterId, userId, flowName, flowRunId, appId);
+ this.entityType = entityType;
+ this.entityId = entityId;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = super.hashCode();
+ result = prime * result + ((entityId == null) ? 0 : entityId.hashCode());
+ result =
+ prime * result + ((entityType == null) ? 0 : entityType.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (!super.equals(obj)) {
+ return false;
+ }
+ TimelineReaderContext other = (TimelineReaderContext) obj;
+ if (entityId == null) {
+ if (other.entityId != null) {
+ return false;
+ }
+ } else if (!entityId.equals(other.entityId)) {
+ return false;
+ }
+ if (entityType == null) {
+ if (other.entityType != null) {
+ return false;
+ }
+ } else if (!entityType.equals(other.entityType)) {
+ return false;
+ }
+ return true;
+ }
+
+ public String getEntityType() {
+ return entityType;
+ }
+
+ public void setEntityType(String type) {
+ this.entityType = type;
+ }
+
+ public String getEntityId() {
+ return entityId;
+ }
+
+ public void setEntityId(String id) {
+ this.entityId = id;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 65d251d..a7d864e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -27,15 +27,22 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+import com.google.common.annotations.VisibleForTesting;
+
@Private
@Unstable
public class TimelineReaderManager extends AbstractService {
+ @VisibleForTesting
+ public static final String UID_KEY = "UID";
private TimelineReader reader;
public TimelineReaderManager(TimelineReader timelineReader) {
@@ -59,13 +66,63 @@ public class TimelineReaderManager extends AbstractService {
return clusterId;
}
+ private static TimelineEntityType getTimelineEntityType(String entityType) {
+ if (entityType == null) {
+ return null;
+ }
+ try {
+ return TimelineEntityType.valueOf(entityType);
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ }
+
+ /**
+ * Fill UID in the info field of entity based on the query(identified by
+ * entity type).
+ * @param entityType Entity type of query.
+ * @param entity Timeline Entity.
+ * @param context Context defining the query.
+ */
+ private static void fillUID(TimelineEntityType entityType,
+ TimelineEntity entity, TimelineReaderContext context) {
+ if (entityType != null) {
+ switch(entityType) {
+ case YARN_FLOW_ACTIVITY:
+ FlowActivityEntity activityEntity = (FlowActivityEntity)entity;
+ context.setUserId(activityEntity.getUser());
+ context.setFlowName(activityEntity.getFlowName());
+ entity.setUID(UID_KEY,
+ TimelineUIDConverter.FLOW_UID.encodeUID(context));
+ return;
+ case YARN_FLOW_RUN:
+ FlowRunEntity runEntity = (FlowRunEntity)entity;
+ context.setFlowRunId(runEntity.getRunId());
+ entity.setUID(UID_KEY,
+ TimelineUIDConverter.FLOWRUN_UID.encodeUID(context));
+ return;
+ case YARN_APPLICATION:
+ context.setAppId(entity.getId());
+ entity.setUID(UID_KEY,
+ TimelineUIDConverter.APPLICATION_UID.encodeUID(context));
+ return;
+ default:
+ break;
+ }
+ }
+ context.setEntityType(entity.getType());
+ context.setEntityId(entity.getId());
+ entity.setUID(UID_KEY,
+ TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context));
+ }
+
/**
* Get a set of entities matching given predicates. The meaning of each
* argument has been documented with {@link TimelineReader#getEntities}.
*
* @see TimelineReader#getEntities
*/
- Set<TimelineEntity> getEntities(String userId, String clusterId,
+ public Set<TimelineEntity> getEntities(String userId, String clusterId,
String flowName, Long flowRunId, String appId, String entityType,
Long limit, Long createdTimeBegin, Long createdTimeEnd,
Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
@@ -73,10 +130,20 @@ public class TimelineReaderManager extends AbstractService {
Set<String> metricFilters, Set<String> eventFilters,
EnumSet<Field> fieldsToRetrieve) throws IOException {
String cluster = getClusterID(clusterId, getConfig());
- return reader.getEntities(userId, cluster, flowName, flowRunId, appId,
+ Set<TimelineEntity> entities =
+ reader.getEntities(userId, cluster, flowName, flowRunId, appId,
entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
null, null, fieldsToRetrieve);
+ if (entities != null) {
+ TimelineEntityType type = getTimelineEntityType(entityType);
+ TimelineReaderContext context = new TimelineReaderContext(cluster, userId,
+ flowName, flowRunId, appId, entityType, null);
+ for (TimelineEntity entity : entities) {
+ fillUID(type, entity, context);
+ }
+ }
+ return entities;
}
/**
@@ -89,7 +156,16 @@ public class TimelineReaderManager extends AbstractService {
String flowName, Long flowRunId, String appId, String entityType,
String entityId, EnumSet<Field> fields) throws IOException {
String cluster = getClusterID(clusterId, getConfig());
- return reader.getEntity(userId, cluster, flowName, flowRunId, appId,
+ TimelineEntity entity =
+ reader.getEntity(userId, cluster, flowName, flowRunId, appId,
entityType, entityId, null, null, fields);
+
+ if (entity != null) {
+ TimelineEntityType type = getTimelineEntityType(entityType);
+ TimelineReaderContext context = new TimelineReaderContext(cluster, userId,
+ flowName, flowRunId, appId, entityType, null);
+ fillUID(type, entity, context);
+ }
+ return entity;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
new file mode 100644
index 0000000..66abbfc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
@@ -0,0 +1,171 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+
+/**
+ * Set of utility methods to be used across timeline reader.
+ */
+final class TimelineReaderUtils {
+ private TimelineReaderUtils() {
+ }
+
+ /**
+ * Split the passed string along the passed delimiter character while looking
+ * for escape char to interpret the splitted parts correctly. For delimiter or
+ * escape character to be interpreted as part of the string, they have to be
+ * escaped by putting an escape character in front.
+ * @param str string to be split.
+ * @param delimiterChar delimiter used for splitting.
+ * @param escapeChar delimiter and escape character will be escaped using this
+ * character.
+ * @return a list of strings after split.
+ * @throws IllegalArgumentException if string is not properly escaped.
+ */
+ static List<String> split(final String str, final char delimiterChar,
+ final char escapeChar) throws IllegalArgumentException {
+ if (str == null) {
+ return null;
+ }
+ int len = str.length();
+ if (len == 0) {
+ return Collections.emptyList();
+ }
+ List<String> list = new ArrayList<String>();
+ // Keeps track of offset of the passed string.
+ int offset = 0;
+ // Indicates start offset from which characters will be copied from original
+ // string to destination string. Resets when an escape or delimiter char is
+ // encountered.
+ int startOffset = 0;
+ StringBuilder builder = new StringBuilder(len);
+ // Iterate over the string till we reach the end.
+ while (offset < len) {
+ if (str.charAt(offset) == escapeChar) {
+ // An escape character must be followed by a delimiter or escape char
+ // but we have reached the end and have no further character to look at.
+ if (offset + 1 >= len) {
+ throw new IllegalArgumentException(
+ "Escape char not properly escaped.");
+ }
+ char nextChar = str.charAt(offset + 1);
+ // Next character must be a delimiter or an escape char.
+ if (nextChar != escapeChar && nextChar != delimiterChar) {
+ throw new IllegalArgumentException(
+ "Escape char or delimiter char not properly escaped.");
+ }
+ // Copy contents from the offset where last escape or delimiter char was
+ // encountered.
+ if (startOffset < offset) {
+ builder.append(str.substring(startOffset, offset));
+ }
+ builder.append(nextChar);
+ offset += 2;
+ // Reset the start offset as an escape char has been encountered.
+ startOffset = offset;
+ continue;
+ } else if (str.charAt(offset) == delimiterChar) {
+ // A delimiter has been encountered without an escape character.
+ // String needs to be split here. Copy remaining chars and add the
+ // string to list.
+ builder.append(str.substring(startOffset, offset));
+ list.add(builder.toString());
+ // Reset the start offset as a delimiter has been encountered.
+ startOffset = ++offset;
+ builder = new StringBuilder(len - offset);
+ continue;
+ }
+ offset++;
+ }
+ // Copy rest of the characters.
+ if (!str.isEmpty()) {
+ builder.append(str.substring(startOffset));
+ }
+ // Add the last part of delimited string to list.
+ list.add(builder.toString());
+ return list;
+ }
+
+ private static String escapeString(final String str, final char delimiterChar,
+ final char escapeChar) {
+ if (str == null) {
+ return null;
+ }
+ int len = str.length();
+ if (len == 0) {
+ return "";
+ }
+ StringBuilder builder = new StringBuilder();
+ // Keeps track of offset of the passed string.
+ int offset = 0;
+ // Indicates start offset from which characters will be copied from original
+ // string to destination string. Resets when an escape or delimiter char is
+ // encountered.
+ int startOffset = 0;
+ // Iterate over the string till we reach the end.
+ while (offset < len) {
+ char charAtOffset = str.charAt(offset);
+ if (charAtOffset == escapeChar || charAtOffset == delimiterChar) {
+ // If an escape or delimiter character is encountered, copy characters
+ // from the offset where escape or delimiter was last encountered.
+ if (startOffset < offset) {
+ builder.append(str.substring(startOffset, offset));
+ }
+ // Append escape char before delimiter/escape char.
+ builder.append(escapeChar).append(charAtOffset);
+ // Reset start offset for copying characters when next escape/delimiter
+ // char is encountered.
+ startOffset = offset + 1;
+ }
+ offset++;
+ }
+ // Copy remaining characters.
+ builder.append(str.substring(startOffset));
+ return builder.toString();
+ }
+
+ /**
+ * Join different strings in the passed string array delimited by passed
+ * delimiter with delimiter and escape character escaped using passed escape
+ * char.
+ * @param strs strings to be joined.
+ * @param delimiterChar delimiter used to join strings.
+ * @param escapeChar escape character used to escape delimiter and escape
+ * char.
+ * @return a single string joined using delimiter and properly escaped.
+ */
+ static String joinAndEscapeStrings(final String[] strs,
+ final char delimiterChar, final char escapeChar) {
+ int len = strs.length;
+ // Escape each string in string array.
+ for (int index = 0; index < len; index++) {
+ if (strs[index] == null) {
+ return null;
+ }
+ strs[index] = escapeString(strs[index], delimiterChar, escapeChar);
+ }
+ // Join the strings after they have been escaped.
+ return StringUtils.join(strs, delimiterChar);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: YARN-4392. ApplicationCreatedEvent
event time resets after RM restart/failover. Contributed by Naganarasimha G R
and Xuan Gong
Posted by gt...@apache.org.
YARN-4392. ApplicationCreatedEvent event time resets after RM
restart/failover. Contributed by Naganarasimha G R and Xuan Gong
(cherry picked from commit 4546c7582b6762c18ba150d80a8976eb51a8290c)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8d798de3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8d798de3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8d798de3
Branch: refs/heads/YARN-2928
Commit: 8d798de3f0e84f14dc9554b0b65f6906679b4128
Parents: 9ceeb2a
Author: Xuan <xg...@apache.org>
Authored: Mon Dec 7 12:24:55 2015 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:17:07 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java | 6 ++++--
.../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 4 +++-
.../server/resourcemanager/rmapp/TestRMAppTransitions.java | 2 --
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d798de3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 9979a59..b15278c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -335,7 +335,8 @@ public class TimelineClientImpl extends TimelineClient {
@Override
protected void serviceStart() throws Exception {
timelineWriter = createTimelineWriter(
- configuration, authUgi, client, resURI);
+ configuration, authUgi, client, constructResURI(getConfig(),
+ getTimelineServiceAddress(), false));
}
protected TimelineWriter createTimelineWriter(Configuration conf,
@@ -613,7 +614,8 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public String toString() {
- return super.toString() + " with timeline server " + resURI
+ return super.toString() + " with timeline server "
+ + constructResURI(getConfig(), getTimelineServiceAddress(), false)
+ " and writer " + timelineWriter;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d798de3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index ca2771a..a7df5a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -897,6 +897,8 @@ public class RMAppImpl implements RMApp, Recoverable {
//TODO recover collector address.
//this.collectorAddr = appState.getCollectorAddr();
+ // send the ATS create Event
+ sendATSCreateEvent(this, this.startTime);
RMAppAttemptImpl preAttempt = null;
for (ApplicationAttemptId attemptId :
new TreeSet<>(appState.attempts.keySet())) {
@@ -1863,7 +1865,7 @@ public class RMAppImpl implements RMApp, Recoverable {
}
return amNodeLabelExpression;
}
-
+
@Override
public CallerContext getCallerContext() {
return callerContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8d798de3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 0b12db0..ba9a477 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
-
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doReturn;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-4446. Refactor reader API for
better extensibility (Varun Saxena via sjlee)
Posted by gt...@apache.org.
YARN-4446. Refactor reader API for better extensibility (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/675612d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/675612d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/675612d4
Branch: refs/heads/YARN-2928
Commit: 675612d4f4564195b272a86b6bf6826634ceea19
Parents: e02094f
Author: Sangjin Lee <sj...@apache.org>
Authored: Wed Feb 3 16:03:55 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:24:20 2016 -0700
----------------------------------------------------------------------
.../reader/TimelineDataToRetrieve.java | 119 ++++++
.../reader/TimelineEntityFilters.java | 187 +++++++++
.../reader/TimelineReaderContext.java | 10 +
.../reader/TimelineReaderManager.java | 44 +-
.../reader/TimelineReaderUtils.java | 4 +-
.../reader/TimelineReaderWebServices.java | 417 ++++++++-----------
.../reader/TimelineReaderWebServicesUtils.java | 68 +++
.../storage/FileSystemTimelineReaderImpl.java | 116 +++---
.../storage/HBaseTimelineReaderImpl.java | 36 +-
.../timelineservice/storage/TimelineReader.java | 234 +++++------
.../storage/reader/ApplicationEntityReader.java | 204 ++++-----
.../reader/FlowActivityEntityReader.java | 59 +--
.../storage/reader/FlowRunEntityReader.java | 101 ++---
.../storage/reader/GenericEntityReader.java | 192 ++++-----
.../storage/reader/TimelineEntityReader.java | 101 ++---
.../reader/TimelineEntityReaderFactory.java | 74 ++--
.../TestFileSystemTimelineReaderImpl.java | 156 ++++---
.../storage/TestHBaseTimelineStorage.java | 252 ++++++-----
.../flow/TestHBaseStorageFlowActivity.java | 33 +-
.../storage/flow/TestHBaseStorageFlowRun.java | 48 ++-
20 files changed, 1367 insertions(+), 1088 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
new file mode 100644
index 0000000..0cc83d7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineDataToRetrieve.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.EnumSet;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+
+/**
+ * Encapsulates information regarding which data to retrieve for each entity
+ * while querying.<br>
+ * Data to retrieve contains the following :<br>
+ * <ul>
+ * <li><b>confsToRetrieve</b> - Used for deciding which configs to return
+ * in response. This is represented as a {@link TimelineFilterList} object
+ * containing {@link TimelinePrefixFilter} objects. These can either be
+ * exact config keys' or prefixes which are then compared against config
+ * keys' to decide configs(inside entities) to return in response. If null
+ * or empty, all configurations will be fetched if fieldsToRetrieve
+ * contains {@link Field#CONFIGS} or {@link Field#ALL}. This should not be
+ * confused with configFilters which is used to decide which entities to
+ * return instead.</li>
+ * <li><b>metricsToRetrieve</b> - Used for deciding which metrics to return
+ * in response. This is represented as a {@link TimelineFilterList} object
+ * containing {@link TimelinePrefixFilter} objects. These can either be
+ * exact metric ids' or prefixes which are then compared against metric
+ * ids' to decide metrics(inside entities) to return in response. If null
+ * or empty, all metrics will be fetched if fieldsToRetrieve contains
+ * {@link Field#METRICS} or {@link Field#ALL}. This should not be confused
+ * with metricFilters which is used to decide which entities to return
+ * instead.</li>
+ * <li><b>fieldsToRetrieve</b> - Specifies which fields of the entity
+ * object to retrieve, see {@link Field}. If null, retrieves 3 fields,
+ * namely entity id, entity type and entity created time. All fields will
+ * be returned if {@link Field#ALL} is specified.</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public class TimelineDataToRetrieve {
+ private TimelineFilterList confsToRetrieve;
+ private TimelineFilterList metricsToRetrieve;
+ private EnumSet<Field> fieldsToRetrieve;
+
+ public TimelineDataToRetrieve() {
+ this(null, null, null);
+ }
+
+ public TimelineDataToRetrieve(TimelineFilterList confs,
+ TimelineFilterList metrics, EnumSet<Field> fields) {
+ this.confsToRetrieve = confs;
+ this.metricsToRetrieve = metrics;
+ this.fieldsToRetrieve = fields;
+
+ if (this.fieldsToRetrieve == null) {
+ this.fieldsToRetrieve = EnumSet.noneOf(Field.class);
+ }
+ }
+
+ public TimelineFilterList getConfsToRetrieve() {
+ return confsToRetrieve;
+ }
+
+ public void setConfsToRetrieve(TimelineFilterList confs) {
+ this.confsToRetrieve = confs;
+ }
+
+ public TimelineFilterList getMetricsToRetrieve() {
+ return metricsToRetrieve;
+ }
+
+ public void setMetricsToRetrieve(TimelineFilterList metrics) {
+ this.metricsToRetrieve = metrics;
+ }
+
+ public EnumSet<Field> getFieldsToRetrieve() {
+ return fieldsToRetrieve;
+ }
+
+ public void setFieldsToRetrieve(EnumSet<Field> fields) {
+ this.fieldsToRetrieve = fields;
+ }
+
+ /**
+ * Adds configs and metrics fields to fieldsToRetrieve(if they are not
+ * present) if confsToRetrieve and metricsToRetrieve are specified.
+ */
+ public void addFieldsBasedOnConfsAndMetricsToRetrieve() {
+ if (!fieldsToRetrieve.contains(Field.CONFIGS) && confsToRetrieve != null &&
+ !confsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.CONFIGS);
+ }
+ if (!fieldsToRetrieve.contains(Field.METRICS) &&
+ metricsToRetrieve != null &&
+ !metricsToRetrieve.getFilterList().isEmpty()) {
+ fieldsToRetrieve.add(Field.METRICS);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
new file mode 100644
index 0000000..5b2c300
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineEntityFilters.java
@@ -0,0 +1,187 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Encapsulates information regarding the filters to apply while querying. These
+ * filters restrict the number of entities to return.<br>
+ * Filters contain the following :<br>
+ * <ul>
+ * <li><b>limit</b> - A limit on the number of entities to return. If null
+ * or {@literal <=0}, defaults to {@link #DEFAULT_LIMIT}.</li>
+ * <li><b>createdTimeBegin</b> - Matched entities should not be created
+ * before this timestamp. If null or {@literal <=0}, defaults to 0.</li>
+ * <li><b>createdTimeEnd</b> - Matched entities should not be created after
+ * this timestamp. If null or {@literal <=0}, defaults to
+ * {@link Long#MAX_VALUE}.</li>
+ * <li><b>relatesTo</b> - Matched entities should relate to given entities.
+ * If null or empty, the relations are not matched.</li>
+ * <li><b>isRelatedTo</b> - Matched entities should be related to given
+ * entities. If null or empty, the relations are not matched.</li>
+ * <li><b>infoFilters</b> - Matched entities should have exact matches to
+ * the given info represented as key-value pairs. If null or empty, the
+ * filter is not applied.</li>
+ * <li><b>configFilters</b> - Matched entities should have exact matches to
+ * the given configs represented as key-value pairs. If null or empty, the
+ * filter is not applied.</li>
+ * <li><b>metricFilters</b> - Matched entities should contain the given
+ * metrics. If null or empty, the filter is not applied.</li>
+ * <li><b>eventFilters</b> - Matched entities should contain the given
+ * events. If null or empty, the filter is not applied.</li>
+ * </ul>
+ */
+@Private
+@Unstable
+public class TimelineEntityFilters {
+ private Long limit;
+ private Long createdTimeBegin;
+ private Long createdTimeEnd;
+ private Map<String, Set<String>> relatesTo;
+ private Map<String, Set<String>> isRelatedTo;
+ private Map<String, Object> infoFilters;
+ private Map<String, String> configFilters;
+ private Set<String> metricFilters;
+ private Set<String> eventFilters;
+ private static final Long DEFAULT_BEGIN_TIME = 0L;
+ private static final Long DEFAULT_END_TIME = Long.MAX_VALUE;
+
+ /**
+ * Default limit of number of entities to return for getEntities API.
+ */
+ public static final long DEFAULT_LIMIT = 100;
+
+ public TimelineEntityFilters() {
+ this(null, null, null, null, null, null, null, null, null);
+ }
+
+ public TimelineEntityFilters(
+ Long entityLimit, Long timeBegin, Long timeEnd,
+ Map<String, Set<String>> entityRelatesTo,
+ Map<String, Set<String>> entityIsRelatedTo,
+ Map<String, Object> entityInfoFilters,
+ Map<String, String> entityConfigFilters,
+ Set<String> entityMetricFilters,
+ Set<String> entityEventFilters) {
+ this.limit = entityLimit;
+ if (this.limit == null || this.limit < 0) {
+ this.limit = DEFAULT_LIMIT;
+ }
+ this.createdTimeBegin = timeBegin;
+ if (this.createdTimeBegin == null || this.createdTimeBegin < 0) {
+ this.createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ this.createdTimeEnd = timeEnd;
+ if (this.createdTimeEnd == null || this.createdTimeEnd < 0) {
+ this.createdTimeEnd = DEFAULT_END_TIME;
+ }
+ this.relatesTo = entityRelatesTo;
+ this.isRelatedTo = entityIsRelatedTo;
+ this.infoFilters = entityInfoFilters;
+ this.configFilters = entityConfigFilters;
+ this.metricFilters = entityMetricFilters;
+ this.eventFilters = entityEventFilters;
+ }
+
+ public Long getLimit() {
+ return limit;
+ }
+
+ public void setLimit(Long entityLimit) {
+ this.limit = entityLimit;
+ if (this.limit == null || this.limit < 0) {
+ this.limit = DEFAULT_LIMIT;
+ }
+ }
+
+ public Long getCreatedTimeBegin() {
+ return createdTimeBegin;
+ }
+
+ public void setCreatedTimeBegin(Long timeBegin) {
+ this.createdTimeBegin = timeBegin;
+ if (this.createdTimeBegin == null || this.createdTimeBegin < 0) {
+ this.createdTimeBegin = DEFAULT_BEGIN_TIME;
+ }
+ }
+
+ public Long getCreatedTimeEnd() {
+ return createdTimeEnd;
+ }
+
+ public void setCreatedTimeEnd(Long timeEnd) {
+ this.createdTimeEnd = timeEnd;
+ if (this.createdTimeEnd == null || this.createdTimeEnd < 0) {
+ this.createdTimeEnd = DEFAULT_END_TIME;
+ }
+ }
+
+ public Map<String, Set<String>> getRelatesTo() {
+ return relatesTo;
+ }
+
+ public void setRelatesTo(Map<String, Set<String>> relations) {
+ this.relatesTo = relations;
+ }
+
+ public Map<String, Set<String>> getIsRelatedTo() {
+ return isRelatedTo;
+ }
+
+ public void setIsRelatedTo(Map<String, Set<String>> relations) {
+ this.isRelatedTo = relations;
+ }
+
+ public Map<String, Object> getInfoFilters() {
+ return infoFilters;
+ }
+
+ public void setInfoFilters(Map<String, Object> filters) {
+ this.infoFilters = filters;
+ }
+
+ public Map<String, String> getConfigFilters() {
+ return configFilters;
+ }
+
+ public void setConfigFilters(Map<String, String> filters) {
+ this.configFilters = filters;
+ }
+
+ public Set<String> getMetricFilters() {
+ return metricFilters;
+ }
+
+ public void setMetricFilters(Set<String> filters) {
+ this.metricFilters = filters;
+ }
+
+ public Set<String> getEventFilters() {
+ return eventFilters;
+ }
+
+ public void setEventFilters(Set<String> filters) {
+ this.eventFilters = filters;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
index d0ad54a..633bb23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderContext.java
@@ -18,11 +18,15 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.server.timelineservice.TimelineContext;
/**
* Encapsulates fields necessary to make a query in timeline reader.
*/
+@Private
+@Unstable
public class TimelineReaderContext extends TimelineContext {
private String entityType;
@@ -34,6 +38,12 @@ public class TimelineReaderContext extends TimelineContext {
this.entityId = entityId;
}
+ public TimelineReaderContext(TimelineReaderContext other) {
+ this(other.getClusterId(), other.getUserId(), other.getFlowName(),
+ other.getFlowRunId(), other.getAppId(), other.getEntityType(),
+ other.getEntityId());
+ }
+
@Override
public int hashCode() {
final int prime = 31;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index a7d864e..84a97ea 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -33,7 +31,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import com.google.common.annotations.VisibleForTesting;
@@ -122,23 +119,14 @@ public class TimelineReaderManager extends AbstractService {
*
* @see TimelineReader#getEntities
*/
- public Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- EnumSet<Field> fieldsToRetrieve) throws IOException {
- String cluster = getClusterID(clusterId, getConfig());
- Set<TimelineEntity> entities =
- reader.getEntities(userId, cluster, flowName, flowRunId, appId,
- entityType, limit, createdTimeBegin, createdTimeEnd, relatesTo,
- isRelatedTo, infoFilters, configFilters, metricFilters, eventFilters,
- null, null, fieldsToRetrieve);
+ public Set<TimelineEntity> getEntities(TimelineReaderContext context,
+ TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
+ throws IOException {
+ context.setClusterId(getClusterID(context.getClusterId(), getConfig()));
+ Set<TimelineEntity> entities = reader.getEntities(
+ new TimelineReaderContext(context), filters, dataToRetrieve);
if (entities != null) {
- TimelineEntityType type = getTimelineEntityType(entityType);
- TimelineReaderContext context = new TimelineReaderContext(cluster, userId,
- flowName, flowRunId, appId, entityType, null);
+ TimelineEntityType type = getTimelineEntityType(context.getEntityType());
for (TimelineEntity entity : entities) {
fillUID(type, entity, context);
}
@@ -152,18 +140,14 @@ public class TimelineReaderManager extends AbstractService {
*
* @see TimelineReader#getEntity
*/
- public TimelineEntity getEntity(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, EnumSet<Field> fields) throws IOException {
- String cluster = getClusterID(clusterId, getConfig());
- TimelineEntity entity =
- reader.getEntity(userId, cluster, flowName, flowRunId, appId,
- entityType, entityId, null, null, fields);
-
+ public TimelineEntity getEntity(TimelineReaderContext context,
+ TimelineDataToRetrieve dataToRetrieve) throws IOException {
+ context.setClusterId(
+ getClusterID(context.getClusterId(), getConfig()));
+ TimelineEntity entity = reader.getEntity(
+ new TimelineReaderContext(context), dataToRetrieve);
if (entity != null) {
- TimelineEntityType type = getTimelineEntityType(entityType);
- TimelineReaderContext context = new TimelineReaderContext(cluster, userId,
- flowName, flowRunId, appId, entityType, null);
+ TimelineEntityType type = getTimelineEntityType(context.getEntityType());
fillUID(type, entity, context);
}
return entity;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
index 66abbfc..c93c631 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderUtils.java
@@ -90,7 +90,7 @@ final class TimelineReaderUtils {
// String needs to be split here. Copy remaining chars and add the
// string to list.
builder.append(str.substring(startOffset, offset));
- list.add(builder.toString());
+ list.add(builder.toString().trim());
// Reset the start offset as a delimiter has been encountered.
startOffset = ++offset;
builder = new StringBuilder(len - offset);
@@ -103,7 +103,7 @@ final class TimelineReaderUtils {
builder.append(str.substring(startOffset));
}
// Add the last part of delimited string to list.
- list.add(builder.toString());
+ list.add(builder.toString().trim());
return list;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index 7a70de8..d68199a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -46,8 +46,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
@@ -70,8 +68,6 @@ public class TimelineReaderWebServices {
@Context private ServletContext ctxt;
- private static final String COMMA_DELIMITER = ",";
- private static final String COLON_DELIMITER = ":";
private static final String QUERY_STRING_SEP = "?";
private static final String RANGE_DELIMITER = "-";
private static final String DATE_PATTERN = "yyyyMMdd";
@@ -243,15 +239,15 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type, id and created time is returned
+ * specified, 3 fields i.e. entity type, id and created time is returned
* (Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances of the given entity type is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances of the given entity type
+ * is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request or UID is incorrect,
- * HTTP 400(Bad Request) is returned.
+ * HTTP 400(Bad Request) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -290,30 +286,14 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
- entities = timelineReaderManager.getEntities(
- TimelineReaderWebServicesUtils.parseStr(context.getUserId()),
- TimelineReaderWebServicesUtils.parseStr(context.getClusterId()),
- TimelineReaderWebServicesUtils.parseStr(context.getFlowName()),
- context.getFlowRunId(),
- TimelineReaderWebServicesUtils.parseStr(context.getAppId()),
- TimelineReaderWebServicesUtils.parseStr(entityType),
- TimelineReaderWebServicesUtils.parseLongStr(limit),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
- TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
- relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
- isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValueObj(
- infofilters, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValueStr(
- conffilters, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseValuesStr(
- metricfilters, COMMA_DELIMITER),
- TimelineReaderWebServicesUtils.parseValuesStr(
- eventfilters, COMMA_DELIMITER),
- TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ context.setEntityType(
+ TimelineReaderWebServicesUtils.parseStr(entityType));
+ entities = timelineReaderManager.getEntities(context,
+ TimelineReaderWebServicesUtils.createTimelineEntityFilters(
+ limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -374,17 +354,17 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type, id, created time is returned
+ * specified, 3 fields i.e. entity type, id, created time is returned
* (Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances of the given entity type is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances of the given entity type
+ * is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow context information cannot be retrieved, HTTP 404(Not Found)
- * is returned.
+ * is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -463,17 +443,17 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type, id, created time is returned
+ * specified, 3 fields i.e. entity type, id, created time is returned
* (Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances of the given entity type is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances of the given entity type
+ * is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow context information cannot be retrieved, HTTP 404(Not Found)
- * is returned.
+ * is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -512,29 +492,14 @@ public class TimelineReaderWebServices {
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
- TimelineReaderWebServicesUtils.parseStr(userId),
- TimelineReaderWebServicesUtils.parseStr(clusterId),
- TimelineReaderWebServicesUtils.parseStr(flowName),
- TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
- TimelineReaderWebServicesUtils.parseStr(appId),
- TimelineReaderWebServicesUtils.parseStr(entityType),
- TimelineReaderWebServicesUtils.parseLongStr(limit),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
- TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
- relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
- isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValueObj(
- infofilters, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValueStr(
- conffilters, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseValuesStr(
- metricfilters, COMMA_DELIMITER),
- TimelineReaderWebServicesUtils.parseValuesStr(
- eventfilters, COMMA_DELIMITER),
- TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ TimelineReaderWebServicesUtils.createTimelineReaderContext(
+ clusterId, userId, flowName, flowRunId, appId, entityType, null),
+
+ TimelineReaderWebServicesUtils.createTimelineEntityFilters(
+ limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -559,16 +524,16 @@ public class TimelineReaderWebServices {
* UID and then used to query backend(Mandatory path param).
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type, id, created time is returned
+ * specified, 3 fields i.e. entity type, id, created time is returned
* (Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link TimelineEntity} instance is returned.
- * On failures,
+ * <cite>TimelineEntity</cite> instance is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request or UID is incorrect,
- * HTTP 400(Bad Request) is returned.
+ * HTTP 400(Bad Request) is returned.<br>
* If entity for the given entity id cannot be found, HTTP 404(Not Found)
- * is returned.
+ * is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -597,11 +562,9 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
- entity = timelineReaderManager.getEntity(context.getUserId(),
- context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
- context.getAppId(), context.getEntityType(), context.getEntityId(),
- TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ entity = timelineReaderManager.getEntity(context,
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -638,16 +601,16 @@ public class TimelineReaderWebServices {
* param).
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type, id, created time is returned
+ * specified, 3 fields i.e. entity type, id, created time is returned
* (Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link TimelineEntity} instance is returned.
- * On failures,
+ * <cite>TimelineEntity</cite> instance is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow context information cannot be retrieved or entity for the given
- * entity id cannot be found, HTTP 404(Not Found) is returned.
+ * entity id cannot be found, HTTP 404(Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -690,16 +653,16 @@ public class TimelineReaderWebServices {
* param).
* @param fields Specifies which fields of the entity object to retrieve, see
* {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type, id and created time is returned
+ * specified, 3 fields i.e. entity type, id and created time is returned
* (Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link TimelineEntity} instance is returned.
- * On failures,
+ * <cite>TimelineEntity</cite> instance is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow context information cannot be retrieved or entity for the given
- * entity id cannot be found, HTTP 404(Not Found) is returned.
+ * entity id cannot be found, HTTP 404(Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -728,17 +691,12 @@ public class TimelineReaderWebServices {
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
- String type = TimelineReaderWebServicesUtils.parseStr(entityType);
- String id = TimelineReaderWebServicesUtils.parseStr(entityId);
try {
entity = timelineReaderManager.getEntity(
- TimelineReaderWebServicesUtils.parseStr(userId),
- TimelineReaderWebServicesUtils.parseStr(clusterId),
- TimelineReaderWebServicesUtils.parseStr(flowName),
- TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
- TimelineReaderWebServicesUtils.parseStr(appId), type, id,
- TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ TimelineReaderWebServicesUtils.createTimelineReaderContext(
+ clusterId, userId, flowName, flowRunId, appId, entityType, entityId),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -746,8 +704,8 @@ public class TimelineReaderWebServices {
if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
(endTime - startTime) + " ms.)");
- throw new NotFoundException("Timeline entity {id: " + id + ", type: " +
- type + " } is not found");
+ throw new NotFoundException("Timeline entity {id: " + entityId +
+ ", type: " + entityType + " } is not found");
}
LOG.info("Processed URL " + url +
" (Took " + (endTime - startTime) + " ms.)");
@@ -765,13 +723,13 @@ public class TimelineReaderWebServices {
* (Mandatory path param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link FlowRunEntity} instance is returned. By default, all metrics for
- * the flow run will be returned.
- * On failures,
+ * <cite>FlowRunEntity</cite> instance is returned. By default, all
+ * metrics for the flow run will be returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request or UID is incorrect,
- * HTTP 400(Bad Request) is returned.
+ * HTTP 400(Bad Request) is returned.<br>
* If flow run for the given flow run id cannot be found, HTTP 404
- * (Not Found) is returned.
+ * (Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -799,9 +757,9 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
- entity = timelineReaderManager.getEntity(context.getUserId(),
- context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
- null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+ context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
+ entity = timelineReaderManager.getEntity(context,
+ new TimelineDataToRetrieve());
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -829,13 +787,13 @@ public class TimelineReaderWebServices {
* @param flowRunId Id of the flow run to be queried(Mandatory path param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link FlowRunEntity} instance is returned. By default, all metrics for
- * the flow run will be returned.
- * On failures,
+ * <cite>FlowRunEntity</cite> instance is returned. By default, all
+ * metrics for the flow run will be returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow run for the given flow run id cannot be found, HTTP 404
- * (Not Found) is returned.
+ * (Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -865,13 +823,13 @@ public class TimelineReaderWebServices {
* @param flowRunId Id of the flow run to be queried(Mandatory path param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link FlowRunEntity} instance is returned. By default, all metrics for
- * the flow run will be returned.
- * On failures,
+ * <cite>FlowRunEntity</cite> instance is returned. By default, all
+ * metrics for the flow run will be returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow run for the given flow run id cannot be found, HTTP 404
- * (Not Found) is returned.
+ * (Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -899,11 +857,10 @@ public class TimelineReaderWebServices {
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
- TimelineReaderWebServicesUtils.parseStr(userId),
- TimelineReaderWebServicesUtils.parseStr(clusterId),
- TimelineReaderWebServicesUtils.parseStr(flowName),
- TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
- null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+ TimelineReaderWebServicesUtils.createTimelineReaderContext(
+ clusterId, userId, flowName, flowRunId, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineDataToRetrieve());
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -930,8 +887,6 @@ public class TimelineReaderWebServices {
* @param uId a delimited string containing clusterid, userid, and flow name
* which are extracted from UID and then used to query backend(Mandatory
* path param).
- * @param flowName Flow name to which the flow runs to be queried belongs to(
- * Mandatory path param).
* @param limit Number of flow runs to return(Optional query param).
* @param createdTimeStart If specified, matched flow runs should not be
* created before this timestamp(Optional query param).
@@ -943,10 +898,11 @@ public class TimelineReaderWebServices {
* other than metrics are returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * set of {@link FlowRunEntity} instances for the given flow are returned.
- * On failures,
+ * set of <cite>FlowRunEntity</cite> instances for the given flow are
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request or UID is incorrect,
- * HTTP 400(Bad Request) is returned.
+ * HTTP 400(Bad Request) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -978,14 +934,13 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
- entities = timelineReaderManager.getEntities(context.getUserId(),
- context.getClusterId(), context.getFlowName(), null, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(),
- TimelineReaderWebServicesUtils.parseLongStr(limit),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
- null, null, null, null, null, null, TimelineReaderWebServicesUtils.
- parseFieldsStr(fields, COMMA_DELIMITER));
+ context.setEntityType(TimelineEntityType.YARN_FLOW_RUN.toString());
+ entities = timelineReaderManager.getEntities(context,
+ TimelineReaderWebServicesUtils.createTimelineEntityFilters(
+ limit, createdTimeStart, createdTimeEnd, null, null, null,
+ null, null, null),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
}
@@ -1019,10 +974,11 @@ public class TimelineReaderWebServices {
* other than metrics are returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * set of {@link FlowRunEntity} instances for the given flow are returned.
- * On failures,
+ * set of <cite>FlowRunEntity</cite> instances for the given flow are
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1064,10 +1020,11 @@ public class TimelineReaderWebServices {
* other than metrics are returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * set of {@link FlowRunEntity} instances for the given flow are returned.
- * On failures,
+ * set of <cite>FlowRunEntity</cite> instances for the given flow are
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1097,15 +1054,14 @@ public class TimelineReaderWebServices {
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
- TimelineReaderWebServicesUtils.parseStr(userId),
- TimelineReaderWebServicesUtils.parseStr(clusterId),
- TimelineReaderWebServicesUtils.parseStr(flowName), null, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(),
- TimelineReaderWebServicesUtils.parseLongStr(limit),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
- null, null, null, null, null, null, TimelineReaderWebServicesUtils.
- parseFieldsStr(fields, COMMA_DELIMITER));
+ TimelineReaderWebServicesUtils.createTimelineReaderContext(
+ clusterId, userId, flowName, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ TimelineReaderWebServicesUtils.createTimelineEntityFilters(
+ limit, createdTimeStart, createdTimeEnd, null, null, null,
+ null, null, null),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
}
@@ -1142,12 +1098,12 @@ public class TimelineReaderWebServices {
* "daterange=-20150711" returns flows active on and before 20150711.
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * set of {@link FlowActivityEntity} instances are returned.
- * On failures,
+ * set of <cite>FlowActivityEntity</cite> instances are returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
- * Error) is returned.
+ * Error) is returned.<br>
*/
@GET
@Path("/flows/")
@@ -1185,10 +1141,10 @@ public class TimelineReaderWebServices {
* "daterange=-20150711" returns flows active on and before 20150711.
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * set of {@link FlowActivityEntity} instances are returned.
- * On failures,
+ * set of <cite>FlowActivityEntity</cite> instances are returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1214,11 +1170,17 @@ public class TimelineReaderWebServices {
Set<TimelineEntity> entities = null;
try {
DateRange range = parseDateRange(dateRange);
+ TimelineEntityFilters entityFilters =
+ TimelineReaderWebServicesUtils.createTimelineEntityFilters(
+ limit, null, null, null, null, null, null, null, null);
+ entityFilters.setCreatedTimeBegin(range.dateStart);
+ entityFilters.setCreatedTimeEnd(range.dateEnd);
entities = timelineReaderManager.getEntities(
- null, TimelineReaderWebServicesUtils.parseStr(clusterId), null, null,
- null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(),
- TimelineReaderWebServicesUtils.parseLongStr(limit), range.dateStart,
- range.dateEnd, null, null, null, null, null, null, null);
+ TimelineReaderWebServicesUtils.createTimelineReaderContext(
+ clusterId, null, null, null, null,
+ TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), null),
+ entityFilters, TimelineReaderWebServicesUtils.
+ createTimelineDataToRetrieve(null, null, null));
} catch (Exception e) {
handleException(e, url, startTime, "limit");
}
@@ -1242,16 +1204,16 @@ public class TimelineReaderWebServices {
* backend(Mandatory path param).
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link TimelineEntity} instance is returned.
- * On failures,
+ * <cite>TimelineEntity</cite> instance is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request or UID is incorrect,
- * HTTP 400(Bad Request) is returned.
+ * HTTP 400(Bad Request) is returned.<br>
* If app for the given app id cannot be found, HTTP 404(Not Found) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1280,11 +1242,10 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
- entity = timelineReaderManager.getEntity(context.getUserId(),
- context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
- context.getAppId(), TimelineEntityType.YARN_APPLICATION.toString(),
- null, TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString());
+ entity = timelineReaderManager.getEntity(context,
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1316,16 +1277,16 @@ public class TimelineReaderWebServices {
* @param userId User id which should match for the app(Optional query param).
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link TimelineEntity} instance is returned.
- * On failures,
+ * <cite>TimelineEntity</cite> instance is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow context information cannot be retrieved or app for the given
- * app id cannot be found, HTTP 404(Not Found) is returned.
+ * app id cannot be found, HTTP 404(Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1361,16 +1322,16 @@ public class TimelineReaderWebServices {
* @param userId User id which should match for the app(Optional query param).
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing a
- * {@link TimelineEntity} instance is returned.
- * On failures,
+ * <cite>TimelineEntity</cite> instance is returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* If flow context information cannot be retrieved or app for the given
- * app id cannot be found, HTTP 404(Not Found) is returned.
+ * app id cannot be found, HTTP 404(Not Found) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1399,14 +1360,11 @@ public class TimelineReaderWebServices {
TimelineEntity entity = null;
try {
entity = timelineReaderManager.getEntity(
- TimelineReaderWebServicesUtils.parseStr(userId),
- TimelineReaderWebServicesUtils.parseStr(clusterId),
- TimelineReaderWebServicesUtils.parseStr(flowName),
- TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
- TimelineReaderWebServicesUtils.parseStr(appId),
- TimelineEntityType.YARN_APPLICATION.toString(), null,
- TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ TimelineReaderWebServicesUtils.createTimelineReaderContext(
+ clusterId, userId, flowName, flowRunId, appId,
+ TimelineEntityType.YARN_APPLICATION.toString(), null),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -1459,15 +1417,15 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances representing apps is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances representing apps is
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request or UID is incorrect,
- * HTTP 400(Bad Request) is returned.
+ * HTTP 400(Bad Request) is returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1505,30 +1463,13 @@ public class TimelineReaderWebServices {
if (context == null) {
throw new BadRequestException("Incorrect UID " + uId);
}
- entities = timelineReaderManager.getEntities(
- TimelineReaderWebServicesUtils.parseStr(context.getUserId()),
- TimelineReaderWebServicesUtils.parseStr(context.getClusterId()),
- TimelineReaderWebServicesUtils.parseStr(context.getFlowName()),
- context.getFlowRunId(),
- TimelineReaderWebServicesUtils.parseStr(context.getAppId()),
- TimelineEntityType.YARN_APPLICATION.toString(),
- TimelineReaderWebServicesUtils.parseLongStr(limit),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
- TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
- TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
- relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
- isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValueObj(
- infofilters, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseKeyStrValueStr(
- conffilters, COMMA_DELIMITER, COLON_DELIMITER),
- TimelineReaderWebServicesUtils.parseValuesStr(
- metricfilters, COMMA_DELIMITER),
- TimelineReaderWebServicesUtils.parseValuesStr(
- eventfilters, COMMA_DELIMITER),
- TimelineReaderWebServicesUtils.parseFieldsStr(
- fields, COMMA_DELIMITER));
+ context.setEntityType(TimelineEntityType.YARN_APPLICATION.toString());
+ entities = timelineReaderManager.getEntities(context,
+ TimelineReaderWebServicesUtils.createTimelineEntityFilters(
+ limit, createdTimeStart, createdTimeEnd, relatesTo, isRelatedTo,
+ infofilters, conffilters, metricfilters, eventfilters),
+ TimelineReaderWebServicesUtils.createTimelineDataToRetrieve(
+ null, null, fields));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -1582,15 +1523,15 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances representing apps is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances representing apps is
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1661,15 +1602,15 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances representing apps is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances representing apps is
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1739,15 +1680,15 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances representing apps is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances representing apps is
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
@@ -1815,15 +1756,15 @@ public class TimelineReaderWebServices {
* events. This is represented as eventfilters=eventid1, eventid2...
* @param fields Specifies which fields of the app entity object to retrieve,
* see {@link Field}. All fields will be retrieved if fields=ALL. If not
- * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * specified, 3 fields i.e. entity type(equivalent to YARN_APPLICATION),
* app id and app created time is returned(Optional query param).
*
* @return If successful, a HTTP 200(OK) response having a JSON representing
- * a set of {@link TimelineEntity} instances representing apps is
- * returned.
- * On failures,
+ * a set of <cite>TimelineEntity</cite> instances representing apps is
+ * returned.<br>
+ * On failures,<br>
* If any problem occurs in parsing request, HTTP 400(Bad Request) is
- * returned.
+ * returned.<br>
* For all other errors while retrieving data, HTTP 500(Internal Server
* Error) is returned.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index d052d51..fc07e51 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -35,10 +35,78 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel
* Set of utility methods to be used by timeline reader web services.
*/
final class TimelineReaderWebServicesUtils {
+ private static final String COMMA_DELIMITER = ",";
+ private static final String COLON_DELIMITER = ":";
+
private TimelineReaderWebServicesUtils() {
}
/**
+ * Parse the passed context information represented as strings and convert
+ * into a {@link TimelineReaderContext} object.
+ * @param clusterId
+ * @param userId
+ * @param flowName
+ * @param flowRunId
+ * @param appId
+ * @param entityType
+ * @param entityId
+ * @return a {@link TimelineReaderContext} object.
+ * @throws Exception
+ */
+ static TimelineReaderContext createTimelineReaderContext(String clusterId,
+ String userId, String flowName, String flowRunId, String appId,
+ String entityType, String entityId) throws Exception {
+ return new TimelineReaderContext(parseStr(clusterId), parseStr(userId),
+ parseStr(flowName), parseLongStr(flowRunId), parseStr(appId),
+ parseStr(entityType), parseStr(entityId));
+ }
+
+ /**
+ * Parse the passed filters represented as strings and convert them into a
+ * {@link TimelineEntityFilters} object.
+ * @param limit
+ * @param createdTimeStart
+ * @param createdTimeEnd
+ * @param relatesTo
+ * @param isRelatedTo
+ * @param infofilters
+ * @param conffilters
+ * @param metricfilters
+ * @param eventfilters
+ * @return a {@link TimelineEntityFilters} object.
+ * @throws Exception
+ */
+ static TimelineEntityFilters createTimelineEntityFilters(String limit,
+ String createdTimeStart, String createdTimeEnd, String relatesTo,
+ String isRelatedTo, String infofilters, String conffilters,
+ String metricfilters, String eventfilters) throws Exception {
+ return new TimelineEntityFilters(parseLongStr(limit),
+ parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd),
+ parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
+ parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
+ parseValuesStr(metricfilters, COMMA_DELIMITER),
+ parseValuesStr(eventfilters, COMMA_DELIMITER));
+ }
+
+ /**
+ * Parse the passed fields represented as strings and convert them into a
+ * {@link TimelineDataToRetrieve} object.
+ * @param confs
+ * @param metrics
+ * @param fields
+ * @return a {@link TimelineDataToRetrieve} object.
+ * @throws Exception
+ */
+ static TimelineDataToRetrieve createTimelineDataToRetrieve(String confs,
+ String metrics, String fields) throws Exception {
+ return new TimelineDataToRetrieve(
+ null, null, parseFieldsStr(fields, COMMA_DELIMITER));
+ }
+
+ /**
* Parse a delimited string and convert it into a set of strings. For
* instance, if delimiter is ",", then the string should be represented as
* "value1,value2,value3".
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index 72c9249..aa0071f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -44,7 +44,9 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.JsonGenerationException;
@@ -264,22 +266,8 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
private Set<TimelineEntity> getEntities(File dir, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) throws IOException {
- if (limit == null || limit <= 0) {
- limit = DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null || createdTimeBegin <= 0) {
- createdTimeBegin = 0L;
- }
- if (createdTimeEnd == null || createdTimeEnd <= 0) {
- createdTimeEnd = Long.MAX_VALUE;
- }
-
+ TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
+ throws IOException {
// First sort the selected entities based on created/start time.
Map<Long, Set<TimelineEntity>> sortedEntities =
new TreeMap<>(
@@ -303,41 +291,48 @@ public class FileSystemTimelineReaderImpl extends AbstractService
if (!entity.getType().equals(entityType)) {
continue;
}
- if (!isTimeInRange(entity.getCreatedTime(), createdTimeBegin,
- createdTimeEnd)) {
+ if (!isTimeInRange(entity.getCreatedTime(),
+ filters.getCreatedTimeBegin(), filters.getCreatedTimeEnd())) {
continue;
}
- if (relatesTo != null && !relatesTo.isEmpty() &&
- !TimelineStorageUtils
- .matchRelations(entity.getRelatesToEntities(), relatesTo)) {
+ if (filters.getRelatesTo() != null &&
+ !filters.getRelatesTo().isEmpty() &&
+ !TimelineStorageUtils.matchRelations(
+ entity.getRelatesToEntities(), filters.getRelatesTo())) {
continue;
}
- if (isRelatedTo != null && !isRelatedTo.isEmpty() &&
- !TimelineStorageUtils
- .matchRelations(entity.getIsRelatedToEntities(), isRelatedTo)) {
+ if (filters.getIsRelatedTo() != null &&
+ !filters.getIsRelatedTo().isEmpty() &&
+ !TimelineStorageUtils.matchRelations(
+ entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) {
continue;
}
- if (infoFilters != null && !infoFilters.isEmpty() &&
- !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
+ if (filters.getInfoFilters() != null &&
+ !filters.getInfoFilters().isEmpty() &&
+ !TimelineStorageUtils.matchFilters(
+ entity.getInfo(), filters.getInfoFilters())) {
continue;
}
- if (configFilters != null && !configFilters.isEmpty() &&
+ if (filters.getConfigFilters() != null &&
+ !filters.getConfigFilters().isEmpty() &&
!TimelineStorageUtils.matchFilters(
- entity.getConfigs(), configFilters)) {
+ entity.getConfigs(), filters.getConfigFilters())) {
continue;
}
- if (metricFilters != null && !metricFilters.isEmpty() &&
+ if (filters.getMetricFilters() != null &&
+ !filters.getMetricFilters().isEmpty() &&
!TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), metricFilters)) {
+ entity.getMetrics(), filters.getMetricFilters())) {
continue;
}
- if (eventFilters != null && !eventFilters.isEmpty() &&
+ if (filters.getEventFilters() != null &&
+ !filters.getEventFilters().isEmpty() &&
!TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), eventFilters)) {
+ entity.getEvents(), filters.getEventFilters())) {
continue;
}
- TimelineEntity entityToBeReturned =
- createEntityToBeReturned(entity, fieldsToRetrieve);
+ TimelineEntity entityToBeReturned = createEntityToBeReturned(
+ entity, dataToRetrieve.getFieldsToRetrieve());
Set<TimelineEntity> entitiesCreatedAtSameTime =
sortedEntities.get(entityToBeReturned.getCreatedTime());
if (entitiesCreatedAtSameTime == null) {
@@ -355,7 +350,7 @@ public class FileSystemTimelineReaderImpl extends AbstractService
for (TimelineEntity entity : entitySet) {
entities.add(entity);
++entitiesAdded;
- if (entitiesAdded >= limit) {
+ if (entitiesAdded >= filters.getLimit()) {
return entities;
}
}
@@ -371,45 +366,40 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
@Override
- public TimelineEntity getEntity(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve)
- throws IOException {
- String flowRunPath = getFlowRunPath(userId, clusterId, flowName,
- flowRunId, appId);
+ public TimelineEntity getEntity(TimelineReaderContext context,
+ TimelineDataToRetrieve dataToRetrieve) throws IOException {
+ String flowRunPath = getFlowRunPath(context.getUserId(),
+ context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId());
File dir = new File(new File(rootPath, ENTITIES_DIR),
- clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
- File entityFile =
- new File(dir, entityId + TIMELINE_SERVICE_STORAGE_EXTENSION);
+ context.getClusterId() + "/" + flowRunPath + "/" + context.getAppId() +
+ "/" + context.getEntityType());
+ File entityFile = new File(
+ dir, context.getEntityId() + TIMELINE_SERVICE_STORAGE_EXTENSION);
try (BufferedReader reader =
new BufferedReader(new InputStreamReader(
new FileInputStream(entityFile), Charset.forName("UTF-8")))) {
TimelineEntity entity = readEntityFromFile(reader);
- return createEntityToBeReturned(entity, fieldsToRetrieve);
+ return createEntityToBeReturned(
+ entity, dataToRetrieve.getFieldsToRetrieve());
} catch (FileNotFoundException e) {
- LOG.info("Cannot find entity {id:" + entityId + " , type:" + entityType +
- "}. Will send HTTP 404 in response.");
+ LOG.info("Cannot find entity {id:" + context.getEntityId() + " , type:" +
+ context.getEntityType() + "}. Will send HTTP 404 in response.");
return null;
}
}
@Override
- public Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) throws IOException {
- String flowRunPath =
- getFlowRunPath(userId, clusterId, flowName, flowRunId, appId);
+ public Set<TimelineEntity> getEntities(TimelineReaderContext context,
+ TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
+ throws IOException {
+ String flowRunPath = getFlowRunPath(context.getUserId(),
+ context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId());
File dir =
new File(new File(rootPath, ENTITIES_DIR),
- clusterId + "/" + flowRunPath + "/" + appId + "/" + entityType);
- return getEntities(dir, entityType, limit, createdTimeBegin, createdTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
+ context.getClusterId() + "/" + flowRunPath + "/" +
+ context.getAppId() + "/" + context.getEntityType());
+ return getEntities(dir, context.getEntityType(), filters, dataToRetrieve);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/675612d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 0ce9a94..9f8257f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -19,8 +19,6 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
import java.util.Set;
import org.apache.commons.logging.Log;
@@ -31,7 +29,9 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
+import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory;
@@ -65,33 +65,21 @@ public class HBaseTimelineReaderImpl
}
@Override
- public TimelineEntity getEntity(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve)
- throws IOException {
+ public TimelineEntity getEntity(TimelineReaderContext context,
+ TimelineDataToRetrieve dataToRetrieve) throws IOException {
TimelineEntityReader reader =
- TimelineEntityReaderFactory.createSingleEntityReader(userId, clusterId,
- flowName, flowRunId, appId, entityType, entityId, confsToRetrieve,
- metricsToRetrieve, fieldsToRetrieve);
+ TimelineEntityReaderFactory.createSingleEntityReader(context,
+ dataToRetrieve);
return reader.readEntity(hbaseConf, conn);
}
@Override
- public Set<TimelineEntity> getEntities(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) throws IOException {
+ public Set<TimelineEntity> getEntities(TimelineReaderContext context,
+ TimelineEntityFilters filters, TimelineDataToRetrieve dataToRetrieve)
+ throws IOException {
TimelineEntityReader reader =
- TimelineEntityReaderFactory.createMultipleEntitiesReader(userId,
- clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters,
- confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
+ TimelineEntityReaderFactory.createMultipleEntitiesReader(context,
+ filters, dataToRetrieve);
return reader.readEntities(hbaseConf, conn);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: MAPREDUCE-6424. Store MR counters
as timeline metrics instead of event. (Naganarasimha G R via varunsaxena)
Posted by gt...@apache.org.
MAPREDUCE-6424. Store MR counters as timeline metrics instead of event. (Naganarasimha G R via varunsaxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/28ec2bee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/28ec2bee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/28ec2bee
Branch: refs/heads/YARN-2928
Commit: 28ec2beeed35c5f30dfde0a62e36b7b3eebcf3b3
Parents: caf23c3
Author: Varun Saxena <va...@apache.org>
Authored: Sun May 1 17:17:24 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:38:05 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 29 +++++++-
.../hadoop/mapreduce/jobhistory/TestEvents.java | 12 +++-
.../mapreduce/jobhistory/AMStartedEvent.java | 12 +++-
.../mapreduce/jobhistory/HistoryEvent.java | 6 ++
.../mapreduce/jobhistory/JobFinishedEvent.java | 20 ++++--
.../jobhistory/JobInfoChangeEvent.java | 10 ++-
.../mapreduce/jobhistory/JobInitedEvent.java | 11 ++-
.../jobhistory/JobPriorityChangeEvent.java | 11 ++-
.../jobhistory/JobQueueChangeEvent.java | 8 +++
.../jobhistory/JobStatusChangedEvent.java | 11 ++-
.../mapreduce/jobhistory/JobSubmittedEvent.java | 10 ++-
.../JobUnsuccessfulCompletionEvent.java | 11 ++-
.../jobhistory/MapAttemptFinishedEvent.java | 12 +++-
.../jobhistory/NormalizedResourceEvent.java | 8 +++
.../jobhistory/ReduceAttemptFinishedEvent.java | 12 +++-
.../jobhistory/TaskAttemptFinishedEvent.java | 11 ++-
.../jobhistory/TaskAttemptStartedEvent.java | 13 +++-
.../TaskAttemptUnsuccessfulCompletionEvent.java | 17 +++--
.../mapreduce/jobhistory/TaskFailedEvent.java | 14 ++--
.../mapreduce/jobhistory/TaskFinishedEvent.java | 13 +++-
.../mapreduce/jobhistory/TaskStartedEvent.java | 8 +++
.../mapreduce/jobhistory/TaskUpdatedEvent.java | 11 ++-
.../mapreduce/util/JobHistoryEventUtils.java | 21 +++++-
.../mapred/TestMRTimelineEventHandling.java | 74 +++++++++++++++++++-
.../org/apache/hadoop/mapred/UtilsForTests.java | 39 +++++++++--
25 files changed, 345 insertions(+), 59 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 639f56e..d32524e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -25,6 +25,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
@@ -68,6 +69,8 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
+import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -1072,6 +1075,15 @@ public class JobHistoryEventHandler extends AbstractService
return entity;
}
+ // create ApplicationEntity with job finished Metrics from HistoryEvent
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createAppEntityWithJobMetrics(HistoryEvent event, JobId jobId) {
+ ApplicationEntity entity = new ApplicationEntity();
+ entity.setId(jobId.getAppId().toString());
+ entity.setMetrics(event.getTimelineMetrics());
+ return entity;
+ }
+
// create BaseEntity from HistoryEvent with adding other info, like:
// timestamp and entityType.
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
@@ -1088,6 +1100,10 @@ public class JobHistoryEventHandler extends AbstractService
if (setCreatedTime) {
entity.setCreatedTime(timestamp);
}
+ Set<TimelineMetric> timelineMetrics = event.getTimelineMetrics();
+ if (timelineMetrics != null) {
+ entity.setMetrics(timelineMetrics);
+ }
return entity;
}
@@ -1203,10 +1219,17 @@ public class JobHistoryEventHandler extends AbstractService
" and handled by timeline service.");
return;
}
+
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ appEntityWithJobMetrics = null;
if (taskId == null) {
// JobEntity
tEntity = createJobEntity(event, timestamp, jobId,
MAPREDUCE_JOB_ENTITY_TYPE, setCreatedTime);
+ if (event.getEventType() == EventType.JOB_FINISHED
+ && event.getTimelineMetrics() != null) {
+ appEntityWithJobMetrics = createAppEntityWithJobMetrics(event, jobId);
+ }
} else {
if (taskAttemptId == null) {
// TaskEntity
@@ -1221,7 +1244,11 @@ public class JobHistoryEventHandler extends AbstractService
}
}
try {
- timelineClient.putEntitiesAsync(tEntity);
+ if (appEntityWithJobMetrics == null) {
+ timelineClient.putEntitiesAsync(tEntity);
+ } else {
+ timelineClient.putEntities(tEntity, appEntityWithJobMetrics);
+ }
} catch (IOException | YarnException e) {
LOG.error("Failed to process Event " + event.getEventType()
+ " for the job : " + jobId, e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 6eb8a43..f5f4a3c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -18,13 +18,15 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.util.ArrayList;
import java.util.Arrays;
-
-import static org.junit.Assert.*;
+import java.util.Set;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
@@ -36,6 +38,7 @@ import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.v2.app.job.impl.JobImpl;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.junit.Test;
public class TestEvents {
@@ -410,6 +413,11 @@ public class TestEvents {
public TimelineEvent toTimelineEvent() {
return null;
}
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
index bbc7090..75e28ec 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/AMStartedEvent.java
@@ -18,16 +18,18 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.avro.util.Utf8;
-
/**
* Event to record start of a task attempt
*
@@ -183,5 +185,9 @@ public class AMStartedEvent implements HistoryEvent {
tEvent.addInfo("START_TIME", getStartTime());
return tEvent;
}
-
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
index 61ce217..cdafe46 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/HistoryEvent.java
@@ -18,9 +18,12 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Interface for event wrapper classes. Implementations each wrap an
@@ -41,4 +44,7 @@ public interface HistoryEvent {
/** Map HistoryEvent to TimelineEvent */
TimelineEvent toTimelineEvent();
+
+ /** Counters or Metrics if any else return null. */
+ Set<TimelineMetric> getTimelineMetrics();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
index 80d3ee6..2efbc57 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobFinishedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -26,6 +28,7 @@ import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record successful completion of job
@@ -148,14 +151,19 @@ public class JobFinishedEvent implements HistoryEvent {
tEvent.addInfo("FAILED_REDUCES", getFailedReduces());
tEvent.addInfo("FINISHED_MAPS", getFinishedMaps());
tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces());
- tEvent.addInfo("MAP_COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getMapCounters()));
- tEvent.addInfo("REDUCE_COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getReduceCounters()));
- tEvent.addInfo("TOTAL_COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getTotalCounters()));
// TODO replace SUCCEEDED with JobState.SUCCEEDED.toString()
tEvent.addInfo("JOB_STATUS", "SUCCEEDED");
return tEvent;
}
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> jobMetrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getMapCounters(), finishTime);
+ jobMetrics.addAll(JobHistoryEventUtils
+ .countersToTimelineMetric(getReduceCounters(), finishTime));
+ jobMetrics.addAll(JobHistoryEventUtils
+ .countersToTimelineMetric(getTotalCounters(), finishTime));
+ return jobMetrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
index ad82443..eb9ad0d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInfoChangeEvent.java
@@ -18,15 +18,15 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import java.io.IOException;
+import java.util.Set;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record changes in the submit and launch time of
@@ -76,4 +76,8 @@ public class JobInfoChangeEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
index 3e0f2f7..d91208c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobInitedEvent.java
@@ -18,13 +18,15 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the initialization of a job
@@ -87,4 +89,9 @@ public class JobInitedEvent implements HistoryEvent {
tEvent.addInfo("UBERIZED", getUberized());
return tEvent;
}
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
index 5deea0a..ac4943e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobPriorityChangeEvent.java
@@ -18,16 +18,16 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import java.io.IOException;
+import java.util.Set;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.JobPriority;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the change of priority of a job
@@ -75,4 +75,9 @@ public class JobPriorityChangeEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
index b9dd359..b12006b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobQueueChangeEvent.java
@@ -18,10 +18,13 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
@SuppressWarnings("deprecation")
public class JobQueueChangeEvent implements HistoryEvent {
@@ -70,4 +73,9 @@ public class JobQueueChangeEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
index a4f2da2..0b33b4f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobStatusChangedEvent.java
@@ -18,15 +18,15 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import java.io.IOException;
+import java.util.Set;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the change of status for a job
@@ -71,4 +71,9 @@ public class JobStatusChangedEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
index 47b2840..07edb58 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
@@ -21,7 +21,9 @@ package org.apache.hadoop.mapreduce.jobhistory;
import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobACL;
@@ -29,8 +31,7 @@ import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the submission of a job
@@ -229,4 +230,9 @@ public class JobSubmittedEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
index ea9798c..5d3c45f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobUnsuccessfulCompletionEvent.java
@@ -18,7 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import com.google.common.base.Joiner;
+import java.util.Collections;
+import java.util.Set;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -26,8 +27,9 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import java.util.Collections;
+import com.google.common.base.Joiner;
/**
* Event to record Failed and Killed completion of jobs
@@ -135,4 +137,9 @@ public class JobUnsuccessfulCompletionEvent implements HistoryEvent {
tEvent.addInfo("FINISHED_REDUCES", getFinishedReduces());
return tEvent;
}
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
index 36737e9..1d28335 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/MapAttemptFinishedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -29,6 +31,7 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record successful completion of a map attempt
@@ -230,8 +233,6 @@ public class MapAttemptFinishedEvent implements HistoryEvent {
tEvent.addInfo("STATUS", getTaskStatus());
tEvent.addInfo("STATE", getState());
tEvent.addInfo("MAP_FINISH_TIME", getMapFinishTime());
- tEvent.addInfo("COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getCounters()));
tEvent.addInfo("HOSTNAME", getHostname());
tEvent.addInfo("PORT", getPort());
tEvent.addInfo("RACK_NAME", getRackName());
@@ -239,5 +240,12 @@ public class MapAttemptFinishedEvent implements HistoryEvent {
"" : getAttemptId().toString());
return tEvent;
}
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> metrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getCounters(), finishTime);
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
index daa454c..8064be9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/NormalizedResourceEvent.java
@@ -17,11 +17,14 @@
*/
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the normalized map/reduce requirements.
@@ -82,4 +85,9 @@ public class NormalizedResourceEvent implements HistoryEvent {
tEvent.addInfo("TASK_TYPE", getTaskType());
return tEvent;
}
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
index 6087c7a..5ddd839 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ReduceAttemptFinishedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -29,6 +31,7 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record successful completion of a reduce attempt
@@ -238,12 +241,17 @@ public class ReduceAttemptFinishedEvent implements HistoryEvent {
tEvent.addInfo("STATE", getState());
tEvent.addInfo("SHUFFLE_FINISH_TIME", getShuffleFinishTime());
tEvent.addInfo("SORT_FINISH_TIME", getSortFinishTime());
- tEvent.addInfo("COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getCounters()));
tEvent.addInfo("HOSTNAME", getHostname());
tEvent.addInfo("PORT", getPort());
tEvent.addInfo("RACK_NAME", getRackName());
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> metrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getCounters(), finishTime);
+ return metrics;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
index c7c4387..30d2b09 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptFinishedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -28,6 +30,7 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record successful task completion
@@ -149,10 +152,14 @@ public class TaskAttemptFinishedEvent implements HistoryEvent {
tEvent.addInfo("FINISH_TIME", getFinishTime());
tEvent.addInfo("STATUS", getTaskStatus());
tEvent.addInfo("STATE", getState());
- tEvent.addInfo("COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getCounters()));
tEvent.addInfo("HOSTNAME", getHostname());
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> metrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getCounters(), finishTime);
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
index d8baec4..20b5d47 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptStartedEvent.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -25,10 +28,9 @@ import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Event to record start of a task attempt
@@ -151,4 +153,9 @@ public class TaskAttemptStartedEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
index 0bb1358..8826c96 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
@@ -18,8 +18,12 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.ProgressSplitsBlock;
import org.apache.hadoop.mapred.TaskStatus;
import org.apache.hadoop.mapreduce.Counters;
import org.apache.hadoop.mapreduce.TaskAttemptID;
@@ -28,10 +32,7 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.hadoop.mapred.ProgressSplitsBlock;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record unsuccessful (Killed/Failed) completion of task attempts
@@ -267,9 +268,13 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
tEvent.addInfo("SHUFFLE_FINISH_TIME", getFinishTime());
tEvent.addInfo("SORT_FINISH_TIME", getFinishTime());
tEvent.addInfo("MAP_FINISH_TIME", getFinishTime());
- tEvent.addInfo("COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getCounters()));
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> metrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getCounters(), finishTime);
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
index 5e82dea..9a29b81 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFailedEvent.java
@@ -18,6 +18,9 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapred.TaskStatus;
@@ -28,8 +31,7 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the failure of a task
@@ -151,9 +153,13 @@ public class TaskFailedEvent implements HistoryEvent {
tEvent.addInfo("ERROR", getError());
tEvent.addInfo("FAILED_ATTEMPT_ID",
getFailedAttemptID() == null ? "" : getFailedAttemptID().toString());
- tEvent.addInfo("COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getCounters()));
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> metrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getCounters(), finishTime);
+ return metrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
index e359e32..fd003cc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskFinishedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -29,6 +31,7 @@ import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the successful completion of a task
@@ -124,8 +127,6 @@ public class TaskFinishedEvent implements HistoryEvent {
TimelineEvent tEvent = new TimelineEvent();
tEvent.setId(StringUtils.toUpperCase(getEventType().name()));
tEvent.addInfo("TASK_TYPE", getTaskType().toString());
- tEvent.addInfo("COUNTERS_GROUPS",
- JobHistoryEventUtils.countersToJSON(getCounters()));
tEvent.addInfo("FINISH_TIME", getFinishTime());
tEvent.addInfo("STATUS", TaskStatus.State.SUCCEEDED.toString());
tEvent.addInfo("SUCCESSFUL_TASK_ATTEMPT_ID",
@@ -133,5 +134,11 @@ public class TaskFinishedEvent implements HistoryEvent {
getSuccessfulTaskAttemptId().toString());
return tEvent;
}
-
+
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ Set<TimelineMetric> jobMetrics = JobHistoryEventUtils
+ .countersToTimelineMetric(getCounters(), finishTime);
+ return jobMetrics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
index d1b97bf..6792ce7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskStartedEvent.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.mapreduce.jobhistory;
+import java.util.Set;
+
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -25,6 +27,7 @@ import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record the start of a task
@@ -84,4 +87,9 @@ public class TaskStartedEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
index b9a389c..0722207 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskUpdatedEvent.java
@@ -18,15 +18,15 @@
package org.apache.hadoop.mapreduce.jobhistory;
-import java.io.IOException;
+import java.util.Set;
+import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-
-import org.apache.avro.util.Utf8;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
/**
* Event to record updates to a task
@@ -71,4 +71,9 @@ public class TaskUpdatedEvent implements HistoryEvent {
return tEvent;
}
+ @Override
+ public Set<TimelineMetric> getTimelineMetrics() {
+ return null;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
index e60380b..225d517 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -17,10 +17,13 @@
*/
package org.apache.hadoop.mapreduce.util;
+import java.util.HashSet;
+import java.util.Set;
+
import org.apache.hadoop.mapreduce.Counter;
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Counters;
-
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.codehaus.jackson.JsonNode;
import org.codehaus.jackson.map.ObjectMapper;
import org.codehaus.jackson.node.ArrayNode;
@@ -53,4 +56,20 @@ public final class JobHistoryEventUtils {
return nodes;
}
+ public static Set<TimelineMetric> countersToTimelineMetric(Counters counters,
+ long timestamp) {
+ Set<TimelineMetric> entityMetrics = new HashSet<TimelineMetric>();
+ for (CounterGroup g : counters) {
+ String groupName = g.getName();
+ for (Counter c : g) {
+ String name = groupName + ":" + c.getName();
+ TimelineMetric metric = new TimelineMetric();
+ metric.setId(name);
+ metric.addValue(timestamp, c.getValue());
+ entityMetrics.add(metric);
+ }
+ }
+ return entityMetrics;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 300b4fb..5915a43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -18,7 +18,12 @@
package org.apache.hadoop.mapred;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
import java.io.File;
+import java.io.FileReader;
import java.io.IOException;
import java.util.EnumSet;
import java.util.List;
@@ -38,11 +43,13 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
+import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Assert;
@@ -200,7 +207,7 @@ public class TestMRTimelineEventHandling {
Assert.assertEquals(apps.size(), 1);
ApplicationReport appReport = apps.get(0);
firstAppId = appReport.getApplicationId();
-
+ UtilsForTests.waitForAppFinished(job, cluster);
checkNewTimelineEvent(firstAppId, appReport);
LOG.info("Run 2nd job which should be failed.");
@@ -213,6 +220,7 @@ public class TestMRTimelineEventHandling {
appReport = apps.get(0).getApplicationId().equals(firstAppId) ?
apps.get(0) : apps.get(1);
+
checkNewTimelineEvent(firstAppId, appReport);
} finally {
@@ -262,6 +270,27 @@ public class TestMRTimelineEventHandling {
Assert.assertTrue("jobEventFilePath: " + jobEventFilePath +
" does not exist.",
jobEventFile.exists());
+ verifyMetricsWhenEvent(jobEventFile, EventType.JOB_FINISHED.name());
+
+ // for this test, we expect MR job metrics are published in YARN_APPLICATION
+ String outputAppDir = basePath + "/YARN_APPLICATION/";
+ entityFolder = new File(outputAppDir);
+ Assert.assertTrue(
+ "Job output directory: " + outputAppDir +
+ " does not exist.",
+ entityFolder.isDirectory());
+
+ // check for job event file
+ String appEventFileName = appId.toString()
+ + FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+
+ String appEventFilePath = outputAppDir + appEventFileName;
+ File appEventFile = new File(appEventFilePath);
+ Assert.assertTrue(
+ "appEventFilePath: " + appEventFilePath +
+ " does not exist.",
+ appEventFile.exists());
+ verifyMetricsWhenEvent(appEventFile, null);
// check for task event file
String outputDirTask = basePath + "/MAPREDUCE_TASK/";
@@ -278,6 +307,7 @@ public class TestMRTimelineEventHandling {
Assert.assertTrue("taskEventFileName: " + taskEventFilePath +
" does not exist.",
taskEventFile.exists());
+ verifyMetricsWhenEvent(taskEventFile, EventType.TASK_FINISHED.name());
// check for task attempt event file
String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/";
@@ -294,6 +324,48 @@ public class TestMRTimelineEventHandling {
File taskAttemptEventFile = new File(taskAttemptEventFilePath);
Assert.assertTrue("taskAttemptEventFileName: " + taskAttemptEventFilePath +
" does not exist.", taskAttemptEventFile.exists());
+ verifyMetricsWhenEvent(taskAttemptEventFile,
+ EventType.MAP_ATTEMPT_FINISHED.name());
+ }
+
+ private void verifyMetricsWhenEvent(File entityFile, String eventId)
+ throws IOException {
+ BufferedReader reader = null;
+ String strLine;
+ try {
+ reader = new BufferedReader(new FileReader(entityFile));
+ boolean jobMetricsFoundForAppEntity = false;
+ while ((strLine = reader.readLine()) != null) {
+ if (strLine.trim().length() > 0) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ FileSystemTimelineReaderImpl.getTimelineRecordFromJSON(
+ strLine.trim(),
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.class);
+ if (eventId == null) {
+ // Job metrics are published without any events for
+ // ApplicationEntity. There is also possibility that some other
+ // ApplicationEntity is published without events, hence loop all
+ if (entity.getEvents().size() == 0) {
+ jobMetricsFoundForAppEntity = entity.getMetrics().size() > 0;
+ if (jobMetricsFoundForAppEntity) {
+ return;
+ }
+ }
+ } else {
+ for (TimelineEvent event : entity.getEvents()) {
+ if (event.getId().equals(eventId)) {
+ assertTrue(entity.getMetrics().size() > 0);
+ return;
+ }
+ }
+ }
+ }
+ }
+ fail("Expected event : " + eventId + " not found in the file "
+ + entityFile);
+ } finally {
+ reader.close();
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/28ec2bee/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
index 972391c..cb494db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.mapred;
+import java.io.DataOutputStream;
import java.io.File;
import java.io.FileInputStream;
-import java.io.DataOutputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
@@ -31,8 +31,10 @@ import java.util.Enumeration;
import java.util.Iterator;
import java.util.List;
import java.util.Properties;
+import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeoutException;
+import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -42,20 +44,24 @@ import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.SequenceFile;
+import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
-import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.io.SequenceFile.CompressionType;
import org.apache.hadoop.mapred.SortValidator.RecordStatsChecker.NonSplitableSequenceFileInputFormat;
import org.apache.hadoop.mapred.lib.IdentityMapper;
import org.apache.hadoop.mapred.lib.IdentityReducer;
import org.apache.hadoop.mapreduce.Cluster.JobTrackerStatus;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
-import org.apache.commons.logging.Log;
+import com.google.common.base.Supplier;
/**
* Utilities used in unit test.
@@ -607,6 +613,29 @@ public class UtilsForTests {
return job;
}
+ public static void waitForAppFinished(RunningJob job, MiniMRYarnCluster cluster)
+ throws IOException {
+ ApplicationId appId = ApplicationId.newInstance(
+ Long.parseLong(job.getID().getJtIdentifier()), job.getID().getId());
+ ConcurrentMap<ApplicationId, RMApp> rmApps =
+ cluster.getResourceManager().getRMContext().getRMApps();
+ if (!rmApps.containsKey(appId)) {
+ throw new IOException("Job not found");
+ }
+ final RMApp rmApp = rmApps.get(appId);
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return RMAppImpl.isAppInFinalState(rmApp);
+ }
+ }, 1000, 1000 * 180);
+ } catch (TimeoutException | InterruptedException e1) {
+ throw new IOException("Yarn application with " + appId + " didn't finish "
+ + "did not reach finale State", e1);
+ }
+ }
+
// Run a job that will be succeeded and wait until it completes
public static RunningJob runJobSucceed(JobConf conf, Path inDir, Path outDir)
throws IOException {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-4712. CPU Usage Metric is not
captured properly in YARN-2928. (Naganarasimha G R via Varun Saxena)
Posted by gt...@apache.org.
YARN-4712. CPU Usage Metric is not captured properly in YARN-2928. (Naganarasimha G R via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1e7f16c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1e7f16c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1e7f16c
Branch: refs/heads/YARN-2928
Commit: b1e7f16cdcd3f6b3f650f44dd7187ee03cfe440d
Parents: 3157d32
Author: Varun Saxena <va...@apache.org>
Authored: Fri Mar 18 23:19:18 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:18 2016 -0700
----------------------------------------------------------------------
.../monitor/ContainersMonitorImpl.java | 9 +-
.../timelineservice/NMTimelinePublisher.java | 21 ++-
.../TestNMTimelinePublisher.java | 157 +++++++++++++++++++
3 files changed, 171 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1e7f16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index da4aac2..e0bd35e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -22,7 +22,6 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;
-import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -32,11 +31,10 @@ import org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -47,6 +45,7 @@ import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
public class ContainersMonitorImpl extends AbstractService implements
@@ -567,8 +566,8 @@ public class ContainersMonitorImpl extends AbstractService implements
NMTimelinePublisher nmMetricsPublisher =
container.getNMTimelinePublisher();
if (nmMetricsPublisher != null) {
- nmMetricsPublisher.reportContainerResourceUsage(container, pId,
- currentPmemUsage, cpuUsageTotalCoresPercentage);
+ nmMetricsPublisher.reportContainerResourceUsage(container,
+ currentPmemUsage, cpuUsagePercentPerCore);
}
} catch (Exception e) {
// Log the exception and proceed to the next container.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1e7f16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 684feaa..70b7e8d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -113,29 +113,28 @@ public class NMTimelinePublisher extends CompositeService {
}
@SuppressWarnings("unchecked")
- public void reportContainerResourceUsage(Container container, String pId,
- Long pmemUsage, Float cpuUsageTotalCoresPercentage) {
+ public void reportContainerResourceUsage(Container container, Long pmemUsage,
+ Float cpuUsagePercentPerCore) {
if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE ||
- cpuUsageTotalCoresPercentage !=
- ResourceCalculatorProcessTree.UNAVAILABLE) {
+ cpuUsagePercentPerCore != ResourceCalculatorProcessTree.UNAVAILABLE) {
ContainerEntity entity =
createContainerEntity(container.getContainerId());
long currentTimeMillis = System.currentTimeMillis();
if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric memoryMetric = new TimelineMetric();
- memoryMetric.setId(ContainerMetric.MEMORY.toString() + pId);
+ memoryMetric.setId(ContainerMetric.MEMORY.toString());
memoryMetric.addValue(currentTimeMillis, pmemUsage);
entity.addMetric(memoryMetric);
}
- if (cpuUsageTotalCoresPercentage !=
- ResourceCalculatorProcessTree.UNAVAILABLE) {
+ if (cpuUsagePercentPerCore != ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
- cpuMetric.setId(ContainerMetric.CPU.toString() + pId);
- cpuMetric.addValue(currentTimeMillis, cpuUsageTotalCoresPercentage);
+ cpuMetric.setId(ContainerMetric.CPU.toString());
+ cpuMetric.addValue(currentTimeMillis,
+ Math.round(cpuUsagePercentPerCore));
entity.addMetric(cpuMetric);
}
- dispatcher.getEventHandler().handle(
- new TimelinePublishEvent(entity, container.getContainerId()
+ dispatcher.getEventHandler()
+ .handle(new TimelinePublishEvent(entity, container.getContainerId()
.getApplicationAttemptId().getApplicationId()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1e7f16c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
new file mode 100644
index 0000000..830ed6b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
@@ -0,0 +1,157 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map.Entry;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.nodemanager.Context;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestNMTimelinePublisher {
+ private static final String MEMORY_ID = "MEMORY";
+ private static final String CPU_ID = "CPU";
+
+ @Test
+ public void testContainerResourceUsage() {
+ Context context = mock(Context.class);
+ @SuppressWarnings("unchecked")
+ ConcurrentMap<ApplicationId, Application> map = mock(ConcurrentMap.class);
+ Application aApp = mock(Application.class);
+ when(map.get(any(ApplicationId.class))).thenReturn(aApp);
+ DummyTimelineClient timelineClient = new DummyTimelineClient();
+ when(aApp.getTimelineClient()).thenReturn(timelineClient);
+ when(context.getApplications()).thenReturn(map);
+ when(context.getNodeId()).thenReturn(NodeId.newInstance("localhost", 0));
+ when(context.getHttpPort()).thenReturn(0);
+ NMTimelinePublisher publisher = new NMTimelinePublisher(context);
+ publisher.init(new Configuration());
+ publisher.start();
+ Container aContainer = mock(Container.class);
+ when(aContainer.getContainerId()).thenReturn(ContainerId.newContainerId(
+ ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1),
+ 0L));
+ publisher.reportContainerResourceUsage(aContainer, 1024L, 8F);
+ verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8);
+ timelineClient.reset();
+
+ publisher.reportContainerResourceUsage(aContainer, 1024L, 0.8F);
+ verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 1);
+ timelineClient.reset();
+
+ publisher.reportContainerResourceUsage(aContainer, 1024L, 0.49F);
+ verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 0);
+ timelineClient.reset();
+
+ publisher.reportContainerResourceUsage(aContainer, 1024L,
+ (float) ResourceCalculatorProcessTree.UNAVAILABLE);
+ verifyPublishedResourceUsageMetrics(timelineClient, 1024L,
+ ResourceCalculatorProcessTree.UNAVAILABLE);
+ publisher.stop();
+ }
+
+ private void verifyPublishedResourceUsageMetrics(
+ DummyTimelineClient timelineClient, long memoryUsage, int cpuUsage) {
+ TimelineEntity[] entities = null;
+ for (int i = 0; i < 10; i++) {
+ entities = timelineClient.getLastPublishedEntities();
+ if (entities != null) {
+ break;
+ }
+ try {
+ Thread.sleep(150L);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ int numberOfResourceMetrics = 0;
+ numberOfResourceMetrics +=
+ (memoryUsage == ResourceCalculatorProcessTree.UNAVAILABLE) ? 0 : 1;
+ numberOfResourceMetrics +=
+ (cpuUsage == ResourceCalculatorProcessTree.UNAVAILABLE) ? 0 : 1;
+ assertNotNull("entities are expected to be published", entities);
+ assertEquals("Expected number of metrics notpublished",
+ numberOfResourceMetrics, entities[0].getMetrics().size());
+ Iterator<TimelineMetric> metrics = entities[0].getMetrics().iterator();
+ while (metrics.hasNext()) {
+ TimelineMetric metric = metrics.next();
+ Iterator<Entry<Long, Number>> entrySet;
+ switch (metric.getId()) {
+ case CPU_ID:
+ if (cpuUsage == ResourceCalculatorProcessTree.UNAVAILABLE) {
+ Assert.fail("Not Expecting CPU Metric to be published");
+ }
+ entrySet = metric.getValues().entrySet().iterator();
+ assertEquals("CPU usage metric not matching", cpuUsage,
+ entrySet.next().getValue());
+ break;
+ case MEMORY_ID:
+ if (memoryUsage == ResourceCalculatorProcessTree.UNAVAILABLE) {
+ Assert.fail("Not Expecting Memory Metric to be published");
+ }
+ entrySet = metric.getValues().entrySet().iterator();
+ assertEquals("Memory usage metric not matching", memoryUsage,
+ entrySet.next().getValue());
+ break;
+ default:
+ Assert.fail("Invalid Resource Usage metric");
+ break;
+ }
+ }
+ }
+
+ protected static class DummyTimelineClient extends TimelineClientImpl {
+ private TimelineEntity[] lastPublishedEntities;
+
+ @Override
+ public void putEntities(TimelineEntity... entities)
+ throws IOException, YarnException {
+ this.lastPublishedEntities = entities;
+ }
+
+ public TimelineEntity[] getLastPublishedEntities() {
+ return lastPublishedEntities;
+ }
+
+ public void reset() {
+ lastPublishedEntities = null;
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: YARN-3816. [Aggregation] App-level
aggregation and accumulation for YARN system metrics (Li Lu via sjlee)
Posted by gt...@apache.org.
YARN-3816. [Aggregation] App-level aggregation and accumulation for YARN system metrics (Li Lu via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64d15d9c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64d15d9c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64d15d9c
Branch: refs/heads/YARN-2928
Commit: 64d15d9cc408072594102adad474f9302234f80c
Parents: 974a9d7
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Apr 22 10:24:40 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:37 2016 -0700
----------------------------------------------------------------------
.../records/timelineservice/TimelineMetric.java | 140 ++++++++++--
.../TimelineMetricCalculator.java | 115 ++++++++++
.../TimelineMetricOperation.java | 167 +++++++++++++++
.../timelineservice/TestTimelineMetric.java | 100 +++++++++
.../TestTimelineServiceRecords.java | 6 +-
.../timelineservice/NMTimelinePublisher.java | 4 +
.../collector/AppLevelTimelineCollector.java | 72 +++++++
.../collector/TimelineCollector.java | 213 ++++++++++++++++++-
.../storage/TimelineAggregationTrack.java | 2 +-
.../collector/TestTimelineCollector.java | 127 +++++++++++
.../TestFileSystemTimelineWriterImpl.java | 43 +++-
.../storage/TestHBaseTimelineStorage.java | 35 ++-
12 files changed, 998 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
index 2f60515..f0c6849 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetric.java
@@ -19,12 +19,13 @@ package org.apache.hadoop.yarn.api.records.timelineservice;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;
-import java.util.Comparator;
+import java.util.Collections;
import java.util.Map;
import java.util.TreeMap;
@@ -48,13 +49,13 @@ public class TimelineMetric {
private Type type;
private String id;
- private Comparator<Long> reverseComparator = new Comparator<Long>() {
- @Override
- public int compare(Long l1, Long l2) {
- return l2.compareTo(l1);
- }
- };
- private TreeMap<Long, Number> values = new TreeMap<>(reverseComparator);
+ // By default, not to do any aggregation operations. This field will NOT be
+ // persisted (like a "transient" member).
+ private TimelineMetricOperation realtimeAggregationOp
+ = TimelineMetricOperation.NOP;
+
+ private TreeMap<Long, Number> values
+ = new TreeMap<>(Collections.reverseOrder());
public TimelineMetric() {
this(Type.SINGLE_VALUE);
@@ -83,6 +84,26 @@ public class TimelineMetric {
this.id = metricId;
}
+ /**
+ * Get the real time aggregation operation of this metric.
+ *
+ * @return Real time aggregation operation
+ */
+ public TimelineMetricOperation getRealtimeAggregationOp() {
+ return realtimeAggregationOp;
+ }
+
+ /**
+ * Set the real time aggregation operation of this metric.
+ *
+ * @param op A timeline metric operation that the metric should perform on
+ * real time aggregations
+ */
+ public void setRealtimeAggregationOp(
+ final TimelineMetricOperation op) {
+ this.realtimeAggregationOp = op;
+ }
+
// required by JAXB
@InterfaceAudience.Private
@XmlElement(name = "values")
@@ -98,8 +119,8 @@ public class TimelineMetric {
if (type == Type.SINGLE_VALUE) {
overwrite(vals);
} else {
- if (values != null) {
- this.values = new TreeMap<Long, Number>(reverseComparator);
+ if (vals != null) {
+ this.values = new TreeMap<>(Collections.reverseOrder());
this.values.putAll(vals);
} else {
this.values = null;
@@ -166,11 +187,100 @@ public class TimelineMetric {
@Override
public String toString() {
- String str = "{id:" + id + ", type:" + type;
- if (!values.isEmpty()) {
- str += ", values:" + values;
+ return "{id: " + id + ", type: " + type +
+ ", realtimeAggregationOp: " +
+ realtimeAggregationOp + "; " + values.toString() +
+ "}";
+ }
+
+ /**
+ * Get the latest timeline metric as single value type.
+ *
+ * @param metric Incoming timeline metric
+ * @return The latest metric in the incoming metric
+ */
+ public static TimelineMetric getLatestSingleValueMetric(
+ TimelineMetric metric) {
+ if (metric.getType() == Type.SINGLE_VALUE) {
+ return metric;
+ } else {
+ TimelineMetric singleValueMetric = new TimelineMetric(Type.SINGLE_VALUE);
+ Long firstKey = metric.values.firstKey();
+ if (firstKey != null) {
+ Number firstValue = metric.values.get(firstKey);
+ singleValueMetric.addValue(firstKey, firstValue);
+ }
+ return singleValueMetric;
}
- str += "}";
- return str;
}
+
+ /**
+ * Get single data timestamp of the metric.
+ *
+ * @return the single data timestamp
+ */
+ public long getSingleDataTimestamp() {
+ if (this.type == Type.SINGLE_VALUE) {
+ if (values.size() == 0) {
+ throw new YarnRuntimeException("Values for this timeline metric is " +
+ "empty.");
+ } else {
+ return values.firstKey();
+ }
+ } else {
+ throw new YarnRuntimeException("Type for this timeline metric is not " +
+ "SINGLE_VALUE.");
+ }
+ }
+
+ /**
+ * Get single data value of the metric.
+ *
+ * @return the single data value
+ */
+ public Number getSingleDataValue() {
+ if (this.type == Type.SINGLE_VALUE) {
+ if (values.size() == 0) {
+ return null;
+ } else {
+ return values.get(values.firstKey());
+ }
+ } else {
+ throw new YarnRuntimeException("Type for this timeline metric is not " +
+ "SINGLE_VALUE.");
+ }
+ }
+
+ /**
+ * Aggregate an incoming metric to the base aggregated metric with the given
+ * operation state in a stateless fashion. The assumption here is
+ * baseAggregatedMetric and latestMetric should be single value data if not
+ * null.
+ *
+ * @param incomingMetric Incoming timeline metric to aggregate
+ * @param baseAggregatedMetric Base timeline metric
+ * @return Result metric after aggregation
+ */
+ public static TimelineMetric aggregateTo(TimelineMetric incomingMetric,
+ TimelineMetric baseAggregatedMetric) {
+ return aggregateTo(incomingMetric, baseAggregatedMetric, null);
+ }
+
+ /**
+ * Aggregate an incoming metric to the base aggregated metric with the given
+ * operation state. The assumption here is baseAggregatedMetric and
+ * latestMetric should be single value data if not null.
+ *
+ * @param incomingMetric Incoming timeline metric to aggregate
+ * @param baseAggregatedMetric Base timeline metric
+ * @param state Operation state
+ * @return Result metric after aggregation
+ */
+ public static TimelineMetric aggregateTo(TimelineMetric incomingMetric,
+ TimelineMetric baseAggregatedMetric, Map<Object, Object> state) {
+ TimelineMetricOperation operation
+ = incomingMetric.getRealtimeAggregationOp();
+ return operation.aggregate(incomingMetric, baseAggregatedMetric, state);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricCalculator.java
new file mode 100644
index 0000000..4c9045f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricCalculator.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+
+/**
+ * A calculator for timeline metrics.
+ */
+public final class TimelineMetricCalculator {
+
+ private TimelineMetricCalculator() {
+ // do nothing.
+ }
+
+ /**
+ * Compare two not-null numbers.
+ * @param n1 Number n1
+ * @param n2 Number n2
+ * @return 0 if n1 equals n2, a negative int if n1 is less than n2, a
+ * positive int otherwise.
+ */
+ public static int compare(Number n1, Number n2) {
+ if (n1 == null || n2 == null) {
+ throw new YarnRuntimeException(
+ "Number to be compared shouldn't be null.");
+ }
+
+ if (n1 instanceof Integer || n1 instanceof Long) {
+ if (n1.longValue() == n2.longValue()) {
+ return 0;
+ } else {
+ return (n1.longValue() < n2.longValue()) ? -1 : 1;
+ }
+ }
+
+ if (n1 instanceof Float || n1 instanceof Double) {
+ if (n1.doubleValue() == n2.doubleValue()) {
+ return 0;
+ } else {
+ return (n1.doubleValue() < n2.doubleValue()) ? -1 : 1;
+ }
+ }
+
+ // TODO throw warnings/exceptions for other types of number.
+ throw new YarnRuntimeException("Unsupported types for number comparison: "
+ + n1.getClass().getName() + ", " + n2.getClass().getName());
+ }
+
+ /**
+ * Subtract operation between two Numbers.
+ * @param n1 Number n1
+ * @param n2 Number n2
+ * @return Number represent to (n1 - n2).
+ */
+ public static Number sub(Number n1, Number n2) {
+ if (n1 == null) {
+ throw new YarnRuntimeException(
+ "Number to be subtracted shouldn't be null.");
+ } else if (n2 == null) {
+ return n1;
+ }
+
+ if (n1 instanceof Integer || n1 instanceof Long) {
+ return n1.longValue() - n2.longValue();
+ }
+
+ if (n1 instanceof Float || n1 instanceof Double) {
+ return n1.doubleValue() - n2.doubleValue();
+ }
+
+ // TODO throw warnings/exceptions for other types of number.
+ return null;
+ }
+
+ /**
+ * Sum up two Numbers.
+ * @param n1 Number n1
+ * @param n2 Number n2
+ * @return Number represent to (n1 + n2).
+ */
+ public static Number sum(Number n1, Number n2) {
+ if (n1 == null) {
+ return n2;
+ } else if (n2 == null) {
+ return n1;
+ }
+
+ if (n1 instanceof Integer || n1 instanceof Long) {
+ return n1.longValue() + n2.longValue();
+ }
+
+ if (n1 instanceof Float || n1 instanceof Double) {
+ return n1.doubleValue() + n2.doubleValue();
+ }
+
+ // TODO throw warnings/exceptions for other types of number.
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricOperation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricOperation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricOperation.java
new file mode 100644
index 0000000..58e5c38
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineMetricOperation.java
@@ -0,0 +1,167 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import java.util.Map;
+
+/**
+ * Aggregation operations.
+ */
+public enum TimelineMetricOperation {
+ NOP("NOP") {
+ /**
+ * Do nothing on the base metric.
+ *
+ * @param incoming Metric a
+ * @param base Metric b
+ * @param state Operation state (not used)
+ * @return Metric b
+ */
+ @Override
+ public TimelineMetric exec(TimelineMetric incoming,
+ TimelineMetric base, Map<Object, Object> state) {
+ return base;
+ }
+ },
+ MAX("MAX") {
+ /**
+ * Keep the greater value of incoming and base. Stateless operation.
+ *
+ * @param incoming Metric a
+ * @param base Metric b
+ * @param state Operation state (not used)
+ * @return the greater value of a and b
+ */
+ @Override
+ public TimelineMetric exec(TimelineMetric incoming,
+ TimelineMetric base, Map<Object, Object> state) {
+ if (base == null) {
+ return incoming;
+ }
+ Number incomingValue = incoming.getSingleDataValue();
+ Number aggregateValue = base.getSingleDataValue();
+ if (aggregateValue == null) {
+ aggregateValue = Long.MIN_VALUE;
+ }
+ if (TimelineMetricCalculator.compare(incomingValue, aggregateValue) > 0) {
+ base.addValue(incoming.getSingleDataTimestamp(), incomingValue);
+ }
+ return base;
+ }
+ },
+ REPLACE("REPLACE") {
+ /**
+ * Replace the base metric with the incoming value. Stateless operation.
+ *
+ * @param incoming Metric a
+ * @param base Metric b
+ * @param state Operation state (not used)
+ * @return Metric a
+ */
+ @Override
+ public TimelineMetric exec(TimelineMetric incoming,
+ TimelineMetric base,
+ Map<Object, Object> state) {
+ return incoming;
+ }
+ },
+ SUM("SUM") {
+ /**
+ * Return the sum of the incoming metric and the base metric if the
+ * operation is stateless. For stateful operations, also subtract the
+ * value of the timeline metric mapped to the PREV_METRIC_STATE_KEY
+ * in the state object.
+ *
+ * @param incoming Metric a
+ * @param base Metric b
+ * @param state Operation state (PREV_METRIC_STATE_KEY's value as Metric p)
+ * @return A metric with value a + b - p
+ */
+ @Override
+ public TimelineMetric exec(TimelineMetric incoming, TimelineMetric base,
+ Map<Object, Object> state) {
+ if (base == null) {
+ return incoming;
+ }
+ Number incomingValue = incoming.getSingleDataValue();
+ Number aggregateValue = base.getSingleDataValue();
+ Number result
+ = TimelineMetricCalculator.sum(incomingValue, aggregateValue);
+
+ // If there are previous value in the state, we will take it off from the
+ // sum
+ if (state != null) {
+ Object prevMetric = state.get(PREV_METRIC_STATE_KEY);
+ if (prevMetric instanceof TimelineMetric) {
+ result = TimelineMetricCalculator.sub(result,
+ ((TimelineMetric) prevMetric).getSingleDataValue());
+ }
+ }
+ base.addValue(incoming.getSingleDataTimestamp(), result);
+ return base;
+ }
+ },
+ AVG("AVERAGE") {
+ /**
+ * Return the average value of the incoming metric and the base metric,
+ * with a given state. Not supported yet.
+ *
+ * @param incoming Metric a
+ * @param base Metric b
+ * @param state Operation state
+ * @return Not finished yet
+ */
+ @Override
+ public TimelineMetric exec(TimelineMetric incoming, TimelineMetric base,
+ Map<Object, Object> state) {
+ // Not supported yet
+ throw new UnsupportedOperationException(
+ "Unsupported aggregation operation: AVERAGE");
+ }
+ };
+
+ public static final String PREV_METRIC_STATE_KEY = "PREV_METRIC";
+
+ /**
+ * Perform the aggregation operation.
+ *
+ * @param incoming Incoming metric
+ * @param aggregate Base aggregation metric
+ * @param state Operation state
+ * @return Result metric for this aggregation operation
+ */
+ public TimelineMetric aggregate(TimelineMetric incoming,
+ TimelineMetric aggregate, Map<Object, Object> state) {
+ return exec(incoming, aggregate, state);
+ }
+
+ private final String opName;
+
+ TimelineMetricOperation(String opString) {
+ opName = opString;
+ }
+
+ @Override
+ public String toString() {
+ return this.opName;
+ }
+
+ abstract TimelineMetric exec(TimelineMetric incoming, TimelineMetric base,
+ Map<Object, Object> state);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineMetric.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineMetric.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineMetric.java
new file mode 100644
index 0000000..3244bc3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineMetric.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.api.records.timelineservice;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric.Type;
+
+import org.junit.Test;
+
+public class TestTimelineMetric {
+
+ @Test
+ public void testTimelineMetricAggregation() {
+ long ts = System.currentTimeMillis();
+ // single_value metric add against null metric
+ TimelineMetric m1 = getSingleValueMetric("MEGA_BYTES_MILLIS",
+ TimelineMetricOperation.SUM, ts, 10000L);
+ TimelineMetric aggregatedMetric = TimelineMetric.aggregateTo(m1, null);
+ assertEquals(10000L, aggregatedMetric.getSingleDataValue());
+
+ TimelineMetric m2 = getSingleValueMetric("MEGA_BYTES_MILLIS",
+ TimelineMetricOperation.SUM, ts, 20000L);
+ aggregatedMetric = TimelineMetric.aggregateTo(m2, aggregatedMetric);
+ assertEquals(30000L, aggregatedMetric.getSingleDataValue());
+
+ // stateful sum test
+ Map<Object, Object> state = new HashMap<>();
+ state.put(TimelineMetricOperation.PREV_METRIC_STATE_KEY, m2);
+ TimelineMetric m2New = getSingleValueMetric("MEGA_BYTES_MILLIS",
+ TimelineMetricOperation.SUM, ts, 10000L);
+ aggregatedMetric = TimelineMetric.aggregateTo(m2New, aggregatedMetric,
+ state);
+ assertEquals(20000L, aggregatedMetric.getSingleDataValue());
+
+ // single_value metric max against single_value metric
+ TimelineMetric m3 = getSingleValueMetric("TRANSFER_RATE",
+ TimelineMetricOperation.MAX, ts, 150L);
+ TimelineMetric aggregatedMax = TimelineMetric.aggregateTo(m3, null);
+ assertEquals(150L, aggregatedMax.getSingleDataValue());
+
+ TimelineMetric m4 = getSingleValueMetric("TRANSFER_RATE",
+ TimelineMetricOperation.MAX, ts, 170L);
+ aggregatedMax = TimelineMetric.aggregateTo(m4, aggregatedMax);
+ assertEquals(170L, aggregatedMax.getSingleDataValue());
+
+ // single_value metric avg against single_value metric
+ TimelineMetric m5 = getSingleValueMetric("TRANSFER_RATE",
+ TimelineMetricOperation.AVG, ts, 150L);
+ try {
+ TimelineMetric.aggregateTo(m5, null);
+ fail("Taking average among metrics is not supported! ");
+ } catch (UnsupportedOperationException e) {
+ // Expected
+ }
+
+ }
+
+ private static TimelineMetric getSingleValueMetric(String id,
+ TimelineMetricOperation op, long timestamp, long value) {
+ TimelineMetric m = new TimelineMetric();
+ m.setId(id);
+ m.setType(Type.SINGLE_VALUE);
+ m.setRealtimeAggregationOp(op);
+ Map<Long, Number> metricValues = new HashMap<Long, Number>();
+ metricValues.put(timestamp, value);
+ m.setValues(metricValues);
+ return m;
+ }
+
+ private static TimelineMetric getTimeSeriesMetric(String id,
+ TimelineMetricOperation op, Map<Long, Number> metricValues) {
+ TimelineMetric m = new TimelineMetric();
+ m.setId(id);
+ m.setType(Type.TIME_SERIES);
+ m.setRealtimeAggregationOp(op);
+ m.setValues(metricValues);
+ return m;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
index 51ec762..592bfa3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/timelineservice/TestTimelineServiceRecords.java
@@ -64,13 +64,13 @@ public class TestTimelineServiceRecords {
metric1.getValues().entrySet().iterator();
Map.Entry<Long, Number> entry = itr.next();
Assert.assertEquals(new Long(3L), entry.getKey());
- Assert.assertEquals(new Double(3.0D), entry.getValue());
+ Assert.assertEquals(3.0D, entry.getValue());
entry = itr.next();
Assert.assertEquals(new Long(2L), entry.getKey());
- Assert.assertEquals(new Integer(2), entry.getValue());
+ Assert.assertEquals(2, entry.getValue());
entry = itr.next();
Assert.assertEquals(new Long(1L), entry.getKey());
- Assert.assertEquals(new Float(1.0F), entry.getValue());
+ Assert.assertEquals(1.0F, entry.getValue());
Assert.assertFalse(itr.hasNext());
entity.addMetric(metric1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 4d3dafd..39a6181 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity.Identifier;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
@@ -119,12 +120,15 @@ public class NMTimelinePublisher extends CompositeService {
if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric memoryMetric = new TimelineMetric();
memoryMetric.setId(ContainerMetric.MEMORY.toString());
+ memoryMetric.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
memoryMetric.addValue(currentTimeMillis, pmemUsage);
entity.addMetric(memoryMetric);
}
if (cpuUsagePercentPerCore != ResourceCalculatorProcessTree.UNAVAILABLE) {
TimelineMetric cpuMetric = new TimelineMetric();
cpuMetric.setId(ContainerMetric.CPU.toString());
+ // TODO: support average
+ cpuMetric.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
cpuMetric.addValue(currentTimeMillis,
Math.round(cpuUsagePercentPerCore));
entity.addMetric(cpuMetric);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index 4fe445a..eb05262 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -18,15 +18,26 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import com.google.common.base.Preconditions;
+import java.util.HashSet;
+import java.util.Set;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
/**
* Service that handles writes to the timeline service and writes them to the
* backing storage for a given YARN application.
@@ -36,8 +47,16 @@ import com.google.common.base.Preconditions;
@Private
@Unstable
public class AppLevelTimelineCollector extends TimelineCollector {
+ private static final Log LOG = LogFactory.getLog(TimelineCollector.class);
+
+ private final static int AGGREGATION_EXECUTOR_NUM_THREADS = 1;
+ private final static int AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS = 15;
+ private static Set<String> entityTypesSkipAggregation
+ = initializeSkipSet();
+
private final ApplicationId appId;
private final TimelineCollectorContext context;
+ private ScheduledThreadPoolExecutor appAggregationExecutor;
public AppLevelTimelineCollector(ApplicationId appId) {
super(AppLevelTimelineCollector.class.getName() + " - " + appId.toString());
@@ -46,6 +65,14 @@ public class AppLevelTimelineCollector extends TimelineCollector {
context = new TimelineCollectorContext();
}
+ private static Set<String> initializeSkipSet() {
+ Set<String> result = new HashSet<>();
+ result.add(TimelineEntityType.YARN_APPLICATION.toString());
+ result.add(TimelineEntityType.YARN_FLOW_RUN.toString());
+ result.add(TimelineEntityType.YARN_FLOW_ACTIVITY.toString());
+ return result;
+ }
+
@Override
protected void serviceInit(Configuration conf) throws Exception {
context.setClusterId(conf.get(YarnConfiguration.RM_CLUSTER_ID,
@@ -60,11 +87,25 @@ public class AppLevelTimelineCollector extends TimelineCollector {
@Override
protected void serviceStart() throws Exception {
+ // Launch the aggregation thread
+ appAggregationExecutor = new ScheduledThreadPoolExecutor(
+ AppLevelTimelineCollector.AGGREGATION_EXECUTOR_NUM_THREADS,
+ new ThreadFactoryBuilder()
+ .setNameFormat("TimelineCollector Aggregation thread #%d")
+ .build());
+ appAggregationExecutor.scheduleAtFixedRate(new AppLevelAggregator(), 0,
+ AppLevelTimelineCollector.AGGREGATION_EXECUTOR_EXEC_INTERVAL_SECS,
+ TimeUnit.SECONDS);
super.serviceStart();
}
@Override
protected void serviceStop() throws Exception {
+ appAggregationExecutor.shutdown();
+ if (!appAggregationExecutor.awaitTermination(10, TimeUnit.SECONDS)) {
+ LOG.info("App-level aggregator shutdown timed out, shutdown now. ");
+ appAggregationExecutor.shutdownNow();
+ }
super.serviceStop();
}
@@ -73,4 +114,35 @@ public class AppLevelTimelineCollector extends TimelineCollector {
return context;
}
+ @Override
+ protected Set<String> getEntityTypesSkipAggregation() {
+ return entityTypesSkipAggregation;
+ }
+
+ private class AppLevelAggregator implements Runnable {
+
+ @Override
+ public void run() {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("App-level real-time aggregating");
+ }
+ try {
+ TimelineCollectorContext currContext = getTimelineEntityContext();
+ TimelineEntity resultEntity = TimelineCollector.aggregateWithoutGroupId(
+ getAggregationGroups(), currContext.getAppId(),
+ TimelineEntityType.YARN_APPLICATION.toString());
+ TimelineEntities entities = new TimelineEntities();
+ entities.addEntity(resultEntity);
+ getWriter().write(currContext.getClusterId(), currContext.getUserId(),
+ currContext.getFlowName(), currContext.getFlowVersion(),
+ currContext.getFlowRunId(), currContext.getAppId(), entities);
+ } catch (Exception e) {
+ LOG.error("Error aggregating timeline metrics", e);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("App-level real-time aggregation complete");
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index 15187d1..8cd645c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -19,6 +19,12 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -27,7 +33,10 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineWriteResponse;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
@@ -41,9 +50,15 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineWriter;
@Private
@Unstable
public abstract class TimelineCollector extends CompositeService {
+
private static final Log LOG = LogFactory.getLog(TimelineCollector.class);
+ public static final String SEPARATOR = "_";
private TimelineWriter writer;
+ private ConcurrentMap<String, AggregationStatusTable> aggregationGroups
+ = new ConcurrentHashMap<>();
+ private static Set<String> entityTypesSkipAggregation
+ = new HashSet<>();
public TimelineCollector(String name) {
super(name);
@@ -68,6 +83,28 @@ public abstract class TimelineCollector extends CompositeService {
this.writer = w;
}
+ protected TimelineWriter getWriter() {
+ return writer;
+ }
+
+ protected Map<String, AggregationStatusTable> getAggregationGroups() {
+ return aggregationGroups;
+ }
+
+ /**
+ * Method to decide the set of timeline entity types the collector should
+ * skip on aggregations. Subclasses may want to override this method to
+ * customize their own behaviors.
+ *
+ * @return A set of strings consists of all types the collector should skip.
+ */
+ protected Set<String> getEntityTypesSkipAggregation() {
+ return entityTypesSkipAggregation;
+ }
+
+ public abstract TimelineCollectorContext getTimelineEntityContext();
+
+
/**
* Handles entity writes. These writes are synchronous and are written to the
* backing storage without buffering/batching. If any entity already exists,
@@ -90,8 +127,12 @@ public abstract class TimelineCollector extends CompositeService {
LOG.debug("putEntities(entities=" + entities + ", callerUgi="
+ callerUgi + ")");
}
-
TimelineCollectorContext context = getTimelineEntityContext();
+
+ // Update application metrics for aggregation
+ updateAggregateStatus(entities, aggregationGroups,
+ getEntityTypesSkipAggregation());
+
return writer.write(context.getClusterId(), context.getUserId(),
context.getFlowName(), context.getFlowVersion(), context.getFlowRunId(),
context.getAppId(), entities);
@@ -117,6 +158,174 @@ public abstract class TimelineCollector extends CompositeService {
}
}
- public abstract TimelineCollectorContext getTimelineEntityContext();
+ /**
+ * Aggregate all metrics in given timeline entities with no predefined states.
+ *
+ * @param entities Entities to aggregate
+ * @param resultEntityId Id of the result entity
+ * @param resultEntityType Type of the result entity
+ * @param needsGroupIdInResult Marks if we want the aggregation group id in
+ * each aggregated metrics.
+ * @return A timeline entity that contains all aggregated TimelineMetric.
+ */
+ public static TimelineEntity aggregateEntities(
+ TimelineEntities entities, String resultEntityId,
+ String resultEntityType, boolean needsGroupIdInResult) {
+ ConcurrentMap<String, AggregationStatusTable> aggregationGroups
+ = new ConcurrentHashMap<>();
+ updateAggregateStatus(entities, aggregationGroups, null);
+ if (needsGroupIdInResult) {
+ return aggregate(aggregationGroups, resultEntityId, resultEntityType);
+ } else {
+ return aggregateWithoutGroupId(
+ aggregationGroups, resultEntityId, resultEntityType);
+ }
+ }
+ /**
+ * Update the aggregation status table for a timeline collector.
+ *
+ * @param entities Entities to update
+ * @param aggregationGroups Aggregation status table
+ * @param typesToSkip Entity types that we can safely assume to skip updating
+ */
+ static void updateAggregateStatus(
+ TimelineEntities entities,
+ ConcurrentMap<String, AggregationStatusTable> aggregationGroups,
+ Set<String> typesToSkip) {
+ for (TimelineEntity e : entities.getEntities()) {
+ if ((typesToSkip != null && typesToSkip.contains(e.getType()))
+ || e.getMetrics().isEmpty()) {
+ continue;
+ }
+ AggregationStatusTable aggrTable = aggregationGroups.get(e.getType());
+ if (aggrTable == null) {
+ AggregationStatusTable table = new AggregationStatusTable();
+ aggrTable = aggregationGroups.putIfAbsent(e.getType(),
+ table);
+ if (aggrTable == null) {
+ aggrTable = table;
+ }
+ }
+ aggrTable.update(e);
+ }
+ }
+
+ /**
+ * Aggregate internal status and generate timeline entities for the
+ * aggregation results.
+ *
+ * @param aggregationGroups Aggregation status table
+ * @param resultEntityId Id of the result entity
+ * @param resultEntityType Type of the result entity
+ * @return A timeline entity that contains all aggregated TimelineMetric.
+ */
+ static TimelineEntity aggregate(
+ Map<String, AggregationStatusTable> aggregationGroups,
+ String resultEntityId, String resultEntityType) {
+ TimelineEntity result = new TimelineEntity();
+ result.setId(resultEntityId);
+ result.setType(resultEntityType);
+ for (Map.Entry<String, AggregationStatusTable> entry
+ : aggregationGroups.entrySet()) {
+ entry.getValue().aggregateAllTo(result, entry.getKey());
+ }
+ return result;
+ }
+
+ /**
+ * Aggregate internal status and generate timeline entities for the
+ * aggregation results. The result metrics will not have aggregation group
+ * information.
+ *
+ * @param aggregationGroups Aggregation status table
+ * @param resultEntityId Id of the result entity
+ * @param resultEntityType Type of the result entity
+ * @return A timeline entity that contains all aggregated TimelineMetric.
+ */
+ static TimelineEntity aggregateWithoutGroupId(
+ Map<String, AggregationStatusTable> aggregationGroups,
+ String resultEntityId, String resultEntityType) {
+ TimelineEntity result = new TimelineEntity();
+ result.setId(resultEntityId);
+ result.setType(resultEntityType);
+ for (Map.Entry<String, AggregationStatusTable> entry
+ : aggregationGroups.entrySet()) {
+ entry.getValue().aggregateAllTo(result, "");
+ }
+ return result;
+ }
+
+ // Note: In memory aggregation is performed in an eventually consistent
+ // fashion.
+ private static class AggregationStatusTable {
+ // On aggregation, for each metric, aggregate all per-entity accumulated
+ // metrics. We only use the id and type for TimelineMetrics in the key set
+ // of this table.
+ private ConcurrentMap<TimelineMetric, Map<String, TimelineMetric>>
+ aggregateTable;
+
+ public AggregationStatusTable() {
+ aggregateTable = new ConcurrentHashMap<>();
+ }
+
+ public void update(TimelineEntity incoming) {
+ String entityId = incoming.getId();
+ for (TimelineMetric m : incoming.getMetrics()) {
+ // Skip if the metric does not need aggregation
+ if (m.getRealtimeAggregationOp() == TimelineMetricOperation.NOP) {
+ continue;
+ }
+ // Update aggregateTable
+ Map<String, TimelineMetric> aggrRow = aggregateTable.get(m);
+ if (aggrRow == null) {
+ Map<String, TimelineMetric> tempRow = new ConcurrentHashMap<>();
+ aggrRow = aggregateTable.putIfAbsent(m, tempRow);
+ if (aggrRow == null) {
+ aggrRow = tempRow;
+ }
+ }
+ aggrRow.put(entityId, m);
+ }
+ }
+
+ public TimelineEntity aggregateTo(TimelineMetric metric, TimelineEntity e,
+ String aggregationGroupId) {
+ if (metric.getRealtimeAggregationOp() == TimelineMetricOperation.NOP) {
+ return e;
+ }
+ Map<String, TimelineMetric> aggrRow = aggregateTable.get(metric);
+ if (aggrRow != null) {
+ TimelineMetric aggrMetric = new TimelineMetric();
+ if (aggregationGroupId.length() > 0) {
+ aggrMetric.setId(metric.getId() + SEPARATOR + aggregationGroupId);
+ } else {
+ aggrMetric.setId(metric.getId());
+ }
+ aggrMetric.setRealtimeAggregationOp(TimelineMetricOperation.NOP);
+ Map<Object, Object> status = new HashMap<>();
+ for (TimelineMetric m : aggrRow.values()) {
+ TimelineMetric.aggregateTo(m, aggrMetric, status);
+ // getRealtimeAggregationOp returns an enum so we can directly
+ // compare with "!=".
+ if (m.getRealtimeAggregationOp()
+ != aggrMetric.getRealtimeAggregationOp()) {
+ aggrMetric.setRealtimeAggregationOp(m.getRealtimeAggregationOp());
+ }
+ }
+ Set<TimelineMetric> metrics = e.getMetrics();
+ metrics.remove(aggrMetric);
+ metrics.add(aggrMetric);
+ }
+ return e;
+ }
+
+ public TimelineEntity aggregateAllTo(TimelineEntity e,
+ String aggregationGroupId) {
+ for (TimelineMetric m : aggregateTable.keySet()) {
+ aggregateTo(m, e, aggregationGroupId);
+ }
+ return e;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
index f0b1e47..6a1e086 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
@@ -24,5 +24,5 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
*
*/
public enum TimelineAggregationTrack {
- FLOW, USER, QUEUE
+ APP, FLOW, USER, QUEUE
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
new file mode 100644
index 0000000..5b4dc50
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/collector/TestTimelineCollector.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
+import org.junit.Test;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+public class TestTimelineCollector {
+
+ private TimelineEntities generateTestEntities(int groups, int entities) {
+ TimelineEntities te = new TimelineEntities();
+ for (int j = 0; j < groups; j++) {
+ for (int i = 0; i < entities; i++) {
+ TimelineEntity entity = new TimelineEntity();
+ String containerId = "container_1000178881110_2002_" + i;
+ entity.setId(containerId);
+ String entityType = "TEST_" + j;
+ entity.setType(entityType);
+ long cTime = 1425016501000L;
+ entity.setCreatedTime(cTime);
+
+ // add metrics
+ Set<TimelineMetric> metrics = new HashSet<>();
+ TimelineMetric m1 = new TimelineMetric();
+ m1.setId("HDFS_BYTES_WRITE");
+ m1.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
+ long ts = System.currentTimeMillis();
+ m1.addValue(ts - 20000, 100L);
+ metrics.add(m1);
+
+ TimelineMetric m2 = new TimelineMetric();
+ m2.setId("VCORES_USED");
+ m2.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
+ m2.addValue(ts - 20000, 3L);
+ metrics.add(m2);
+
+ // m3 should not show up in the aggregation
+ TimelineMetric m3 = new TimelineMetric();
+ m3.setId("UNRELATED_VALUES");
+ m3.addValue(ts - 20000, 3L);
+ metrics.add(m3);
+
+ TimelineMetric m4 = new TimelineMetric();
+ m4.setId("TXN_FINISH_TIME");
+ m4.setRealtimeAggregationOp(TimelineMetricOperation.MAX);
+ m4.addValue(ts - 20000, i);
+ metrics.add(m4);
+
+ entity.addMetrics(metrics);
+ te.addEntity(entity);
+ }
+ }
+
+ return te;
+ }
+
+ @Test
+ public void testAggregation() throws Exception {
+ // Test aggregation with multiple groups.
+ int groups = 3;
+ int n = 50;
+ TimelineEntities testEntities = generateTestEntities(groups, n);
+ TimelineEntity resultEntity = TimelineCollector.aggregateEntities(
+ testEntities, "test_result", "TEST_AGGR", true);
+ assertEquals(resultEntity.getMetrics().size(), groups * 3);
+
+ for (int i = 0; i < groups; i++) {
+ Set<TimelineMetric> metrics = resultEntity.getMetrics();
+ for (TimelineMetric m : metrics) {
+ if (m.getId().startsWith("HDFS_BYTES_WRITE")) {
+ assertEquals(100 * n, m.getSingleDataValue().intValue());
+ } else if (m.getId().startsWith("VCORES_USED")) {
+ assertEquals(3 * n, m.getSingleDataValue().intValue());
+ } else if (m.getId().startsWith("TXN_FINISH_TIME")) {
+ assertEquals(n - 1, m.getSingleDataValue());
+ } else {
+ fail("Unrecognized metric! " + m.getId());
+ }
+ }
+ }
+
+ // Test aggregation with a single group.
+ TimelineEntities testEntities1 = generateTestEntities(1, n);
+ TimelineEntity resultEntity1 = TimelineCollector.aggregateEntities(
+ testEntities1, "test_result", "TEST_AGGR", false);
+ assertEquals(resultEntity1.getMetrics().size(), 3);
+
+ Set<TimelineMetric> metrics = resultEntity1.getMetrics();
+ for (TimelineMetric m : metrics) {
+ if (m.getId().equals("HDFS_BYTES_WRITE")) {
+ assertEquals(100 * n, m.getSingleDataValue().intValue());
+ } else if (m.getId().equals("VCORES_USED")) {
+ assertEquals(3 * n, m.getSingleDataValue().intValue());
+ } else if (m.getId().equals("TXN_FINISH_TIME")) {
+ assertEquals(n - 1, m.getSingleDataValue());
+ } else {
+ fail("Unrecognized metric! " + m.getId());
+ }
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index 5ce7d3b..2f79daa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -25,11 +25,15 @@ import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Test;
@@ -51,6 +55,26 @@ public class TestFileSystemTimelineWriterImpl {
entity.setCreatedTime(1425016501000L);
te.addEntity(entity);
+ TimelineMetric metric = new TimelineMetric();
+ String metricId = "CPU";
+ metric.setId(metricId);
+ metric.setType(TimelineMetric.Type.SINGLE_VALUE);
+ metric.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
+ metric.addValue(1425016501000L, 1234567L);
+
+ TimelineEntity entity2 = new TimelineEntity();
+ String id2 = "metric";
+ String type2 = "app";
+ entity2.setId(id2);
+ entity2.setType(type2);
+ entity2.setCreatedTime(1425016503000L);
+ entity2.addMetric(metric);
+ te.addEntity(entity2);
+
+ Map<String, TimelineMetric> aggregatedMetrics =
+ new HashMap<String, TimelineMetric>();
+ aggregatedMetrics.put(metricId, metric);
+
FileSystemTimelineWriterImpl fsi = null;
try {
fsi = new FileSystemTimelineWriterImpl();
@@ -68,11 +92,27 @@ public class TestFileSystemTimelineWriterImpl {
assertTrue(f.exists() && !f.isDirectory());
List<String> data = Files.readAllLines(path, StandardCharsets.UTF_8);
// ensure there's only one entity + 1 new line
- assertTrue(data.size() == 2);
+ assertTrue("data size is:" + data.size(), data.size() == 2);
String d = data.get(0);
// confirm the contents same as what was written
assertEquals(d, TimelineUtils.dumpTimelineRecordtoJSON(entity));
+ // verify aggregated metrics
+ String fileName2 = fsi.getOutputRoot() +
+ "/entities/cluster_id/user_id/flow_name/flow_version/12345678/app_id/"
+ + type2 + "/" + id2 +
+ FileSystemTimelineWriterImpl.TIMELINE_SERVICE_STORAGE_EXTENSION;
+ Path path2 = Paths.get(fileName2);
+ File file = new File(fileName2);
+ assertTrue(file.exists() && !file.isDirectory());
+ List<String> data2 = Files.readAllLines(path2, StandardCharsets.UTF_8);
+ // ensure there's only one entity + 1 new line
+ assertTrue("data size is:" + data.size(), data2.size() == 2);
+ String metricToString = data2.get(0);
+ // confirm the contents same as what was written
+ assertEquals(metricToString,
+ TimelineUtils.dumpTimelineRecordtoJSON(entity2));
+
// delete the directory
File outputDir = new File(fsi.getOutputRoot());
FileUtils.deleteDirectory(outputDir);
@@ -84,4 +124,5 @@ public class TestFileSystemTimelineWriterImpl {
}
}
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64d15d9c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 6b57ec4..8ab54bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetricOperation;
import org.apache.hadoop.yarn.api.records.timelineservice.ApplicationEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -539,6 +540,26 @@ public class TestHBaseTimelineStorage {
metrics.add(m1);
entity.addMetrics(metrics);
+ // add aggregated metrics
+ TimelineEntity aggEntity = new TimelineEntity();
+ String type = TimelineEntityType.YARN_APPLICATION.toString();
+ aggEntity.setId(appId);
+ aggEntity.setType(type);
+ long cTime2 = 1425016502000L;
+ long mTime2 = 1425026902000L;
+ aggEntity.setCreatedTime(cTime2);
+
+ TimelineMetric aggMetric = new TimelineMetric();
+ aggMetric.setId("MEM_USAGE");
+ Map<Long, Number> aggMetricValues = new HashMap<Long, Number>();
+ ts = System.currentTimeMillis();
+ aggMetricValues.put(ts - 120000, 102400000);
+ aggMetric.setType(Type.SINGLE_VALUE);
+ aggMetric.setRealtimeAggregationOp(TimelineMetricOperation.SUM);
+ aggMetric.setValues(aggMetricValues);
+ Set<TimelineMetric> aggMetrics = new HashSet<>();
+ aggMetrics.add(aggMetric);
+ entity.addMetrics(aggMetrics);
te.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
@@ -564,7 +585,7 @@ public class TestHBaseTimelineStorage {
Result result = new ApplicationTable().getResult(c1, conn, get);
assertTrue(result != null);
- assertEquals(15, result.size());
+ assertEquals(16, result.size());
// check the row key
byte[] row1 = result.getRow();
@@ -652,10 +673,17 @@ public class TestHBaseTimelineStorage {
assertEquals(conf, conf2);
Set<TimelineMetric> metrics2 = e1.getMetrics();
- assertEquals(metrics, metrics2);
+ assertEquals(2, metrics2.size());
for (TimelineMetric metric2 : metrics2) {
Map<Long, Number> metricValues2 = metric2.getValues();
- matchMetrics(metricValues, metricValues2);
+ assertTrue(metric2.getId().equals("MAP_SLOT_MILLIS") ||
+ metric2.getId().equals("MEM_USAGE"));
+ if (metric2.getId().equals("MAP_SLOT_MILLIS")) {
+ matchMetrics(metricValues, metricValues2);
+ }
+ if (metric2.getId().equals("MEM_USAGE")) {
+ matchMetrics(aggMetricValues, metricValues2);
+ }
}
} finally {
if (hbi != null) {
@@ -724,7 +752,6 @@ public class TestHBaseTimelineStorage {
m1.setValues(metricValues);
metrics.add(m1);
entity.addMetrics(metrics);
-
te.addEntity(entity);
HBaseTimelineWriterImpl hbi = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters
in TimelineReader (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index 9793ce6..3b8036d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -154,6 +154,14 @@ class TestFlowDataGenerator {
metrics.add(m2);
entity.addMetrics(metrics);
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ long endTs = 1439379885000L;
+ event.setTimestamp(endTs);
+ String expKey = "foo_event_greater";
+ String expVal = "test_app_greater";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
return entity;
}
@@ -178,6 +186,14 @@ class TestFlowDataGenerator {
m1.setValues(metricValues);
metrics.add(m1);
entity.addMetrics(metrics);
+ TimelineEvent event = new TimelineEvent();
+ event.setId(ApplicationMetricsConstants.CREATED_EVENT_TYPE);
+ long endTs = 1439379885000L;
+ event.setTimestamp(endTs);
+ String expKey = "foo_event_greater";
+ String expVal = "test_app_greater";
+ event.addInfo(expKey, expVal);
+ entity.addEvent(event);
return entity;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index f04dd48..a724db2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -47,8 +47,10 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineReaderImpl;
import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriterImpl;
@@ -307,7 +309,7 @@ public class TestHBaseStorageFlowRun {
assertEquals(141L, Bytes.toLong(values.get(q)));
// check metric2
- assertEquals(2, values.size());
+ assertEquals(3, values.size());
q = ColumnHelper.getColumnQualifier(
FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(), metric2);
assertTrue(values.containsKey(q));
@@ -318,11 +320,10 @@ public class TestHBaseStorageFlowRun {
@Test
public void testWriteFlowRunMetricsPrefix() throws Exception {
- String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
- String user = "testWriteFlowRunMetricsOneFlow_user1";
- String flow = "testing_flowRun_metrics_flow_name";
+ String cluster = "testWriteFlowRunMetricsPrefix_cluster1";
+ String user = "testWriteFlowRunMetricsPrefix_user1";
+ String flow = "testWriteFlowRunMetricsPrefix_flow_name";
String flowVersion = "CF7022C10F1354";
- long runid = 1002345678919L;
TimelineEntities te = new TimelineEntities();
TimelineEntity entityApp1 = TestFlowDataGenerator
@@ -335,33 +336,30 @@ public class TestHBaseStorageFlowRun {
hbi = new HBaseTimelineWriterImpl(c1);
hbi.init(c1);
String appName = "application_11111111111111_1111";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.write(cluster, user, flow, flowVersion, 1002345678919L, appName, te);
// write another application with same metric to this flow
te = new TimelineEntities();
TimelineEntity entityApp2 = TestFlowDataGenerator
.getEntityMetricsApp2(System.currentTimeMillis());
te.addEntity(entityApp2);
appName = "application_11111111111111_2222";
- hbi.write(cluster, user, flow, flowVersion, runid, appName, te);
+ hbi.write(cluster, user, flow, flowVersion, 1002345678918L, appName, te);
hbi.flush();
} finally {
hbi.close();
}
- // check flow run
- checkFlowRunTable(cluster, user, flow, runid, c1);
-
// use the timeline reader to verify data
HBaseTimelineReaderImpl hbr = null;
try {
hbr = new HBaseTimelineReaderImpl();
hbr.init(c1);
hbr.start();
- TimelineFilterList metricsToRetrieve =
- new TimelineFilterList(new TimelinePrefixFilter(TimelineCompareOp.EQUAL,
- metric1.substring(0, metric1.indexOf("_") + 1)));
+ TimelineFilterList metricsToRetrieve = new TimelineFilterList(
+ Operator.OR, new TimelinePrefixFilter(TimelineCompareOp.EQUAL,
+ metric1.substring(0, metric1.indexOf("_") + 1)));
TimelineEntity entity = hbr.getEntity(
- new TimelineReaderContext(cluster, user, flow, runid, null,
+ new TimelineReaderContext(cluster, user, flow, 1002345678919L, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null),
new TimelineDataToRetrieve(null, metricsToRetrieve, null));
assertTrue(TimelineEntityType.YARN_FLOW_RUN.matches(entity.getType()));
@@ -377,7 +375,7 @@ public class TestHBaseStorageFlowRun {
}
switch (id) {
case metric1:
- assertEquals(141L, value);
+ assertEquals(40L, value);
break;
default:
fail("unrecognized metric: " + id);
@@ -385,31 +383,16 @@ public class TestHBaseStorageFlowRun {
}
Set<TimelineEntity> entities = hbr.getEntities(
- new TimelineReaderContext(cluster, user, flow, runid, null,
+ new TimelineReaderContext(cluster, user, flow, null, null,
TimelineEntityType.YARN_FLOW_RUN.toString(), null),
new TimelineEntityFilters(),
new TimelineDataToRetrieve(null, metricsToRetrieve, null));
- assertEquals(1, entities.size());
+ assertEquals(2, entities.size());
+ int metricCnt = 0;
for (TimelineEntity timelineEntity : entities) {
- Set<TimelineMetric> timelineMetrics = timelineEntity.getMetrics();
- assertEquals(1, timelineMetrics.size());
- for (TimelineMetric metric : timelineMetrics) {
- String id = metric.getId();
- Map<Long, Number> values = metric.getValues();
- assertEquals(1, values.size());
- Number value = null;
- for (Number n : values.values()) {
- value = n;
- }
- switch (id) {
- case metric1:
- assertEquals(141L, value);
- break;
- default:
- fail("unrecognized metric: " + id);
- }
- }
+ metricCnt += timelineEntity.getMetrics().size();
}
+ assertEquals(2, metricCnt);
} finally {
hbr.close();
}
@@ -417,9 +400,9 @@ public class TestHBaseStorageFlowRun {
@Test
public void testWriteFlowRunsMetricFields() throws Exception {
- String cluster = "testWriteFlowRunMetricsOneFlow_cluster1";
- String user = "testWriteFlowRunMetricsOneFlow_user1";
- String flow = "testing_flowRun_metrics_flow_name";
+ String cluster = "testWriteFlowRunsMetricFields_cluster1";
+ String user = "testWriteFlowRunsMetricFields_user1";
+ String flow = "testWriteFlowRunsMetricFields_flow_name";
String flowVersion = "CF7022C10F1354";
long runid = 1002345678919L;
@@ -592,6 +575,214 @@ public class TestHBaseStorageFlowRun {
}
}
+ @Test
+ public void testFilterFlowRunsByCreatedTime() throws Exception {
+ String cluster = "cluster2";
+ String user = "user2";
+ String flow = "flow_name2";
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(
+ System.currentTimeMillis());
+ entityApp1.setCreatedTime(1425016501000L);
+ te.addEntity(entityApp1);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
+ "application_11111111111111_1111", te);
+ // write another application with same metric to this flow
+ te = new TimelineEntities();
+ TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(
+ System.currentTimeMillis());
+ entityApp2.setCreatedTime(1425016502000L);
+ te.addEntity(entityApp2);
+ hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678918L,
+ "application_11111111111111_2222", te);
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+
+ // use the timeline reader to verify data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow,
+ null, null, TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, 1425016501000L, 1425016502001L, null,
+ null, null, null, null, null), new TimelineDataToRetrieve());
+ assertEquals(2, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("user2@flow_name2/1002345678918") &&
+ !entity.getId().equals("user2@flow_name2/1002345678919")) {
+ fail("Entities with flow runs 1002345678918 and 1002345678919" +
+ "should be present.");
+ }
+ }
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, 1425016501050L, null, null, null,
+ null, null, null, null), new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("user2@flow_name2/1002345678918")) {
+ fail("Entity with flow run 1002345678918 should be present.");
+ }
+ }
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, null, 1425016501050L, null, null,
+ null, null, null, null), new TimelineDataToRetrieve());
+ assertEquals(1, entities.size());
+ for (TimelineEntity entity : entities) {
+ if (!entity.getId().equals("user2@flow_name2/1002345678919")) {
+ fail("Entity with flow run 1002345678919 should be present.");
+ }
+ }
+ } finally {
+ hbr.close();
+ }
+ }
+
+ @Test
+ public void testMetricFilters() throws Exception {
+ String cluster = "cluster1";
+ String user = "user1";
+ String flow = "flow_name1";
+
+ TimelineEntities te = new TimelineEntities();
+ TimelineEntity entityApp1 = TestFlowDataGenerator.getEntityMetricsApp1(
+ System.currentTimeMillis());
+ te.addEntity(entityApp1);
+
+ HBaseTimelineWriterImpl hbi = null;
+ Configuration c1 = util.getConfiguration();
+ try {
+ hbi = new HBaseTimelineWriterImpl(c1);
+ hbi.init(c1);
+ hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678919L,
+ "application_11111111111111_1111", te);
+ // write another application with same metric to this flow
+ te = new TimelineEntities();
+ TimelineEntity entityApp2 = TestFlowDataGenerator.getEntityMetricsApp2(
+ System.currentTimeMillis());
+ te.addEntity(entityApp2);
+ hbi.write(cluster, user, flow, "CF7022C10F1354", 1002345678918L,
+ "application_11111111111111_2222", te);
+ hbi.flush();
+ } finally {
+ hbi.close();
+ }
+
+ // use the timeline reader to verify data
+ HBaseTimelineReaderImpl hbr = null;
+ try {
+ hbr = new HBaseTimelineReaderImpl();
+ hbr.init(c1);
+ hbr.start();
+
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, metric1, 101));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, metric1, 43));
+ list2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.EQUAL, metric2, 57));
+ TimelineFilterList metricFilterList =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ Set<TimelineEntity> entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null,
+ null, TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(2, entities.size());
+ int metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ }
+ assertEquals(3, metricCnt);
+
+ TimelineFilterList metricFilterList1 = new TimelineFilterList(
+ new TimelineCompareFilter(
+ TimelineCompareOp.LESS_OR_EQUAL, metric1, 127),
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, metric2, 30));
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList1, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(1, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ }
+ assertEquals(2, metricCnt);
+
+ TimelineFilterList metricFilterList2 = new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN, metric1, 32),
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL, metric2, 57));
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList2, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList metricFilterList3 = new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.EQUAL, "s_metric", 32));
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList3, null),
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.METRICS)));
+ assertEquals(0, entities.size());
+
+ TimelineFilterList list3 = new TimelineFilterList();
+ list3.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, metric1, 101));
+ TimelineFilterList list4 = new TimelineFilterList();
+ list4.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, metric1, 43));
+ list4.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.EQUAL, metric2, 57));
+ TimelineFilterList metricFilterList4 =
+ new TimelineFilterList(Operator.OR, list3, list4);
+ TimelineFilterList metricsToRetrieve = new TimelineFilterList(Operator.OR,
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL,
+ metric2.substring(0, metric2.indexOf("_") + 1)));
+ entities = hbr.getEntities(
+ new TimelineReaderContext(cluster, user, flow, null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(), null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList4, null),
+ new TimelineDataToRetrieve(null, metricsToRetrieve,
+ EnumSet.of(Field.ALL)));
+ assertEquals(2, entities.size());
+ metricCnt = 0;
+ for (TimelineEntity entity : entities) {
+ metricCnt += entity.getMetrics().size();
+ }
+ assertEquals(1, metricCnt);
+ } finally {
+ hbr.close();
+ }
+ }
+
@AfterClass
public static void tearDownAfterClass() throws Exception {
util.shutdownMiniCluster();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index ace218b..2738e6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -148,7 +148,7 @@ public class TestHBaseStorageFlowRunCompaction {
}
// check flow run for one flow many apps
- checkFlowRunTable(cluster, user, flow, runid, c1, 3);
+ checkFlowRunTable(cluster, user, flow, runid, c1, 4);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
new file mode 100644
index 0000000..36d0d7b
--- /dev/null
+++ b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineExistsFilter.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Filter class which represents filter to be applied based on existence of a
+ * value.
+ */
+@Private
+@Unstable
+public class TimelineExistsFilter extends TimelineFilter {
+
+ private final TimelineCompareOp compareOp;
+ private final String value;
+
+ public TimelineExistsFilter(TimelineCompareOp op, String value) {
+ this.value = value;
+ if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("CompareOp for exists filter should " +
+ "be EQUAL or NOT_EQUAL");
+ }
+ this.compareOp = op;
+ }
+
+ @Override
+ public TimelineFilterType getFilterType() {
+ return TimelineFilterType.EXISTS;
+ }
+
+ public String getValue() {
+ return value;
+ }
+
+ public TimelineCompareOp getCompareOp() {
+ return compareOp;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s (%s %s)",
+ this.getClass().getSimpleName(), this.compareOp.name(), this.value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
new file mode 100644
index 0000000..58f0ee9
--- /dev/null
+++ b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValueFilter.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Filter class which represents filter to be applied based on key-value pair
+ * being equal or not to the values in back-end store.
+ */
+@Private
+@Unstable
+public class TimelineKeyValueFilter extends TimelineCompareFilter {
+ public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val,
+ boolean keyMustExistFlag) {
+ super(op, key, val, keyMustExistFlag);
+ if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("TimelineCompareOp for equality"
+ + " filter should be EQUAL or NOT_EQUAL");
+ }
+ }
+
+ public TimelineKeyValueFilter(TimelineCompareOp op, String key, Object val) {
+ this(op, key, val, true);
+ }
+
+ @Override
+ public TimelineFilterType getFilterType() {
+ return TimelineFilterType.KEY_VALUE;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
new file mode 100644
index 0000000..0d34d47
--- /dev/null
+++ b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineKeyValuesFilter.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader.filter;
+
+import java.util.Set;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Filter class which represents filter to be applied based on multiple values
+ * for a key and these values being equal or not equal to values in back-end
+ * store.
+ */
+@Private
+@Unstable
+public class TimelineKeyValuesFilter extends TimelineFilter {
+ private final TimelineCompareOp compareOp;
+ private final String key;
+ private final Set<Object> values;
+ public TimelineKeyValuesFilter(TimelineCompareOp op, String key,
+ Set<Object> values) {
+ if (op != TimelineCompareOp.EQUAL && op != TimelineCompareOp.NOT_EQUAL) {
+ throw new IllegalArgumentException("TimelineCompareOp for multi value "
+ + "equality filter should be EQUAL or NOT_EQUAL");
+ }
+ this.compareOp = op;
+ this.key = key;
+ this.values = values;
+ }
+
+ @Override
+ public TimelineFilterType getFilterType() {
+ return TimelineFilterType.KEY_VALUES;
+ }
+
+ public String getKey() {
+ return key;
+ }
+
+ public Set<Object> getValues() {
+ return values;
+ }
+
+ public TimelineCompareOp getCompareOp() {
+ return compareOp;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s (%s, %s:%s)",
+ this.getClass().getSimpleName(), this.compareOp.name(),
+ this.key, (values == null) ? "" : values.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
new file mode 100644
index 0000000..4099e92
--- /dev/null
+++ b/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineEntityFiltersType.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.storage.common;
+
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilter.TimelineFilterType;
+
+/**
+ * Used to define which filter to match.
+ */
+enum TimelineEntityFiltersType {
+ CONFIG {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUE;
+ }
+ },
+ INFO {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUE;
+ }
+ },
+ METRIC {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.COMPARE;
+ }
+ },
+ EVENT {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.EXISTS;
+ }
+ },
+ IS_RELATED_TO {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUES;
+ }
+ },
+ RELATES_TO {
+ boolean isValidFilter(TimelineFilterType filterType) {
+ return filterType == TimelineFilterType.LIST ||
+ filterType == TimelineFilterType.KEY_VALUES;
+ }
+ };
+
+ /**
+ * Checks whether filter type is valid for the filter being matched.
+ *
+ * @param filterType filter type.
+ * @return true, if its a valid filter, false otherwise.
+ */
+ abstract boolean isValidFilter(TimelineFilterType filterType);
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: MAPREDUCE-6546. reconcile the two
versions of the timeline service performance tests. (Sangjin Lee via
Naganarasimha G R)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cd444089/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
index 92e0b14..4c42bd3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/test/MapredTestDriver.java
@@ -18,6 +18,19 @@
package org.apache.hadoop.test;
+import org.apache.hadoop.fs.DFSCIOTest;
+import org.apache.hadoop.fs.DistributedFSCheck;
+import org.apache.hadoop.fs.JHLogAnalyzer;
+import org.apache.hadoop.fs.TestDFSIO;
+import org.apache.hadoop.fs.TestFileSystem;
+import org.apache.hadoop.fs.loadGenerator.DataGenerator;
+import org.apache.hadoop.fs.loadGenerator.LoadGenerator;
+import org.apache.hadoop.fs.loadGenerator.LoadGeneratorMR;
+import org.apache.hadoop.fs.loadGenerator.StructureGenerator;
+import org.apache.hadoop.fs.slive.SliveTest;
+import org.apache.hadoop.hdfs.NNBench;
+import org.apache.hadoop.hdfs.NNBenchWithoutMR;
+import org.apache.hadoop.io.FileBench;
import org.apache.hadoop.io.TestSequenceFile;
import org.apache.hadoop.mapred.BigMapOutput;
import org.apache.hadoop.mapred.GenericMRLoadGenerator;
@@ -28,28 +41,13 @@ import org.apache.hadoop.mapred.TestMapRed;
import org.apache.hadoop.mapred.TestSequenceFileInputFormat;
import org.apache.hadoop.mapred.TestTextInputFormat;
import org.apache.hadoop.mapred.ThreadedMapBenchmark;
-import org.apache.hadoop.mapreduce.TimelineServicePerformance;
-import org.apache.hadoop.mapred.TimelineServicePerformanceV2;
import org.apache.hadoop.mapreduce.FailJob;
import org.apache.hadoop.mapreduce.LargeSorter;
import org.apache.hadoop.mapreduce.MiniHadoopClusterManager;
import org.apache.hadoop.mapreduce.SleepJob;
+import org.apache.hadoop.mapreduce.TimelineServicePerformance;
import org.apache.hadoop.util.ProgramDriver;
-import org.apache.hadoop.hdfs.NNBench;
-import org.apache.hadoop.hdfs.NNBenchWithoutMR;
-import org.apache.hadoop.fs.TestFileSystem;
-import org.apache.hadoop.fs.TestDFSIO;
-import org.apache.hadoop.fs.DFSCIOTest;
-import org.apache.hadoop.fs.DistributedFSCheck;
-import org.apache.hadoop.io.FileBench;
-import org.apache.hadoop.fs.JHLogAnalyzer;
-import org.apache.hadoop.fs.loadGenerator.DataGenerator;
-import org.apache.hadoop.fs.loadGenerator.LoadGenerator;
-import org.apache.hadoop.fs.loadGenerator.LoadGeneratorMR;
-import org.apache.hadoop.fs.loadGenerator.StructureGenerator;
-import org.apache.hadoop.fs.slive.SliveTest;
-
/**
* Driver for Map-reduce tests.
*
@@ -93,9 +91,8 @@ public class MapredTestDriver {
pgd.addClass("sleep", SleepJob.class,
"A job that sleeps at each map and reduce task.");
pgd.addClass("timelineperformance", TimelineServicePerformance.class,
- "A job that launches mappers to test timlineserver performance.");
- pgd.addClass("timelineperformance", TimelineServicePerformanceV2.class,
- "A job that launch mappers to test timline service v.2 performance.");
+ "A job that launches mappers to test timline service " +
+ "performance.");
pgd.addClass("nnbench", NNBench.class,
"A benchmark that stresses the namenode w/ MR.");
pgd.addClass("nnbenchWithoutMR", NNBenchWithoutMR.class,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: YARN-4986. Add a check in the
coprocessor for table to operated on (Vrushali C via sjlee)
Posted by gt...@apache.org.
YARN-4986. Add a check in the coprocessor for table to operated on (Vrushali C via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/084a334a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/084a334a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/084a334a
Branch: refs/heads/YARN-2928
Commit: 084a334ae52179e2f14bdadab5b7c6d5b447be01
Parents: 64d15d9
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Apr 29 17:13:32 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:40 2016 -0700
----------------------------------------------------------------------
.../storage/common/TimelineStorageUtils.java | 20 +++++++
.../storage/entity/EntityTable.java | 2 +-
.../storage/flow/FlowRunCoprocessor.java | 39 +++++++++++--
.../storage/flow/FlowScanner.java | 13 +++--
.../storage/flow/TestHBaseStorageFlowRun.java | 61 ++++++++++++++++++++
.../flow/TestHBaseStorageFlowRunCompaction.java | 36 ++++++++++++
6 files changed, 160 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
index 2d85bab..18f975a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/common/TimelineStorageUtils.java
@@ -32,8 +32,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.client.Result;
@@ -56,6 +58,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Fiel
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationCompactionDimension;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.AggregationOperation;
import org.apache.hadoop.yarn.server.timelineservice.storage.flow.Attribute;
+import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
import org.apache.hadoop.yarn.util.ConverterUtils;
/**
@@ -887,4 +890,21 @@ public final class TimelineStorageUtils {
Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
entity.addEvents(eventsSet);
}
+
+ public static boolean isFlowRunTable(HRegionInfo hRegionInfo,
+ Configuration conf) {
+ String regionTableName = hRegionInfo.getTable().getNameAsString();
+ String flowRunTableName = conf.get(FlowRunTable.TABLE_NAME_CONF_NAME,
+ FlowRunTable.DEFAULT_TABLE_NAME);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("regionTableName=" + regionTableName);
+ }
+ if (flowRunTableName.equalsIgnoreCase(regionTableName)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(" table is the flow run table!! " + flowRunTableName);
+ }
+ return true;
+ }
+ return false;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
index 3e3e3ab..b194f07 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/entity/EntityTable.java
@@ -84,7 +84,7 @@ public class EntityTable extends BaseTable<EntityTable> {
+ ".table.metrics.ttl";
/** default value for entity table name. */
- private static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
+ public static final String DEFAULT_TABLE_NAME = "timelineservice.entity";
/** default TTL is 30 days for metrics timeseries. */
private static final int DEFAULT_METRICS_TTL = 2592000;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
index 450640a..8ea51a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowRunCoprocessor.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimestampGen
public class FlowRunCoprocessor extends BaseRegionObserver {
private static final Log LOG = LogFactory.getLog(FlowRunCoprocessor.class);
+ private boolean isFlowRunRegion = false;
private HRegion region;
/**
@@ -70,9 +71,15 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
if (e instanceof RegionCoprocessorEnvironment) {
RegionCoprocessorEnvironment env = (RegionCoprocessorEnvironment) e;
this.region = env.getRegion();
+ isFlowRunRegion = TimelineStorageUtils.isFlowRunTable(
+ region.getRegionInfo(), env.getConfiguration());
}
}
+ public boolean isFlowRunRegion() {
+ return isFlowRunRegion;
+ }
+
/*
* (non-Javadoc)
*
@@ -93,6 +100,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
WALEdit edit, Durability durability) throws IOException {
Map<String, byte[]> attributes = put.getAttributesMap();
+ if (!isFlowRunRegion) {
+ return;
+ }
// Assumption is that all the cells in a put are the same operation.
List<Tag> tags = new ArrayList<>();
if ((attributes != null) && (attributes.size() > 0)) {
@@ -160,6 +170,10 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
@Override
public void preGetOp(ObserverContext<RegionCoprocessorEnvironment> e,
Get get, List<Cell> results) throws IOException {
+ if (!isFlowRunRegion) {
+ return;
+ }
+
Scan scan = new Scan(get);
scan.setMaxVersions();
RegionScanner scanner = null;
@@ -190,11 +204,14 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
@Override
public RegionScanner preScannerOpen(
ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
- RegionScanner s) throws IOException {
- // set max versions for scan to see all
- // versions to aggregate for metrics
- scan.setMaxVersions();
- return s;
+ RegionScanner scanner) throws IOException {
+
+ if (isFlowRunRegion) {
+ // set max versions for scan to see all
+ // versions to aggregate for metrics
+ scan.setMaxVersions();
+ }
+ return scanner;
}
/*
@@ -213,6 +230,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
public RegionScanner postScannerOpen(
ObserverContext<RegionCoprocessorEnvironment> e, Scan scan,
RegionScanner scanner) throws IOException {
+ if (!isFlowRunRegion) {
+ return scanner;
+ }
return new FlowScanner(e.getEnvironment(), scan.getBatch(),
scanner, FlowScannerOperation.READ);
}
@@ -221,6 +241,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
public InternalScanner preFlush(
ObserverContext<RegionCoprocessorEnvironment> c, Store store,
InternalScanner scanner) throws IOException {
+ if (!isFlowRunRegion) {
+ return scanner;
+ }
if (LOG.isDebugEnabled()) {
if (store != null) {
LOG.debug("preFlush store = " + store.getColumnFamilyName()
@@ -241,6 +264,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
@Override
public void postFlush(ObserverContext<RegionCoprocessorEnvironment> c,
Store store, StoreFile resultFile) {
+ if (!isFlowRunRegion) {
+ return;
+ }
if (LOG.isDebugEnabled()) {
if (store != null) {
LOG.debug("postFlush store = " + store.getColumnFamilyName()
@@ -262,6 +288,9 @@ public class FlowRunCoprocessor extends BaseRegionObserver {
InternalScanner scanner, ScanType scanType, CompactionRequest request)
throws IOException {
+ if (!isFlowRunRegion) {
+ return scanner;
+ }
FlowScannerOperation requestOp = FlowScannerOperation.MINOR_COMPACTION;
if (request != null) {
requestOp = (request.isMajor() ? FlowScannerOperation.MAJOR_COMPACTION
http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
index 0ace529..398d7b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/FlowScanner.java
@@ -210,7 +210,7 @@ class FlowScanner implements RegionScanner, Closeable {
if (comp.compare(currentColumnQualifier, newColumnQualifier) != 0) {
if (converter != null && isNumericConverter(converter)) {
addedCnt += emitCells(cells, currentColumnCells, currentAggOp,
- (NumericValueConverter)converter, currentTimestamp);
+ converter, currentTimestamp);
}
resetState(currentColumnCells, alreadySeenAggDim);
currentColumnQualifier = newColumnQualifier;
@@ -219,6 +219,7 @@ class FlowScanner implements RegionScanner, Closeable {
}
// No operation needs to be performed on non numeric converters.
if (!isNumericConverter(converter)) {
+ currentColumnCells.add(cell);
nextCell(cellLimit);
continue;
}
@@ -228,7 +229,7 @@ class FlowScanner implements RegionScanner, Closeable {
}
if (!currentColumnCells.isEmpty()) {
addedCnt += emitCells(cells, currentColumnCells, currentAggOp,
- (NumericValueConverter)converter, currentTimestamp);
+ converter, currentTimestamp);
if (LOG.isDebugEnabled()) {
if (addedCnt > 0) {
LOG.debug("emitted cells. " + addedCnt + " for " + this.action
@@ -345,7 +346,7 @@ class FlowScanner implements RegionScanner, Closeable {
* parameter.
*/
private int emitCells(List<Cell> cells, SortedSet<Cell> currentColumnCells,
- AggregationOperation currentAggOp, NumericValueConverter converter,
+ AggregationOperation currentAggOp, ValueConverter converter,
long currentTimestamp) throws IOException {
if ((currentColumnCells == null) || (currentColumnCells.size() == 0)) {
return 0;
@@ -372,12 +373,14 @@ class FlowScanner implements RegionScanner, Closeable {
cells.addAll(currentColumnCells);
return currentColumnCells.size();
case READ:
- Cell sumCell = processSummation(currentColumnCells, converter);
+ Cell sumCell = processSummation(currentColumnCells,
+ (NumericValueConverter) converter);
cells.add(sumCell);
return 1;
case MAJOR_COMPACTION:
List<Cell> finalCells = processSummationMajorCompaction(
- currentColumnCells, converter, currentTimestamp);
+ currentColumnCells, (NumericValueConverter) converter,
+ currentTimestamp);
cells.addAll(finalCells);
return finalCells.size();
default:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index a724db2..801d43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -19,18 +19,21 @@
package org.apache.hadoop.yarn.server.timelineservice.storage.flow;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.IOException;
import java.util.EnumSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Get;
@@ -38,6 +41,8 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.regionserver.HRegionServer;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities;
@@ -57,6 +62,8 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.HBaseTimelineWriter
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineSchemaCreator;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnHelper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
+import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -84,6 +91,60 @@ public class TestHBaseStorageFlowRun {
TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
}
+ @Test
+ public void checkCoProcessorOff() throws IOException, InterruptedException {
+ Configuration hbaseConf = util.getConfiguration();
+ TableName table = TableName.valueOf(hbaseConf.get(
+ FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Connection conn = null;
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ Admin admin = conn.getAdmin();
+ if (admin == null) {
+ throw new IOException("Can't check tables since admin is null");
+ }
+ if (admin.tableExists(table)) {
+ // check the regions.
+ // check in flow run table
+ util.waitUntilAllRegionsAssigned(table);
+ HRegionServer server = util.getRSForFirstRegionInTable(table);
+ List<HRegion> regions = server.getOnlineRegions(table);
+ for (HRegion region : regions) {
+ assertTrue(TimelineStorageUtils.isFlowRunTable(region.getRegionInfo(),
+ hbaseConf));
+ }
+ }
+
+ table = TableName.valueOf(hbaseConf.get(
+ FlowActivityTable.TABLE_NAME_CONF_NAME,
+ FlowActivityTable.DEFAULT_TABLE_NAME));
+ if (admin.tableExists(table)) {
+ // check the regions.
+ // check in flow activity table
+ util.waitUntilAllRegionsAssigned(table);
+ HRegionServer server = util.getRSForFirstRegionInTable(table);
+ List<HRegion> regions = server.getOnlineRegions(table);
+ for (HRegion region : regions) {
+ assertFalse(TimelineStorageUtils.isFlowRunTable(region.getRegionInfo(),
+ hbaseConf));
+ }
+ }
+
+ table = TableName.valueOf(hbaseConf.get(
+ EntityTable.TABLE_NAME_CONF_NAME,
+ EntityTable.DEFAULT_TABLE_NAME));
+ if (admin.tableExists(table)) {
+ // check the regions.
+ // check in entity run table
+ util.waitUntilAllRegionsAssigned(table);
+ HRegionServer server = util.getRSForFirstRegionInTable(table);
+ List<HRegion> regions = server.getOnlineRegions(table);
+ for (HRegion region : regions) {
+ assertFalse(TimelineStorageUtils.isFlowRunTable(region.getRegionInfo(),
+ hbaseConf));
+ }
+ }
+ }
+
/**
* Writes 4 timeline entities belonging to one flow run through the
* {@link HBaseTimelineWriterImpl}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/084a334a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
index 2738e6a..e7e7ba4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRunCompaction.java
@@ -39,6 +39,8 @@ import org.apache.hadoop.hbase.Tag;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
@@ -87,6 +89,40 @@ public class TestHBaseStorageFlowRunCompaction {
TimelineSchemaCreator.createAllTables(util.getConfiguration(), false);
}
+ /** writes non numeric data into flow run table
+ * reads it back
+ *
+ * @throws Exception
+ */
+ @Test
+ public void testWriteNonNumericData() throws Exception {
+ String rowKey = "nonNumericRowKey";
+ String column = "nonNumericColumnName";
+ String value = "nonNumericValue";
+ byte[] rowKeyBytes = Bytes.toBytes(rowKey);
+ byte[] columnNameBytes = Bytes.toBytes(column);
+ byte[] valueBytes = Bytes.toBytes(value);
+ Put p = new Put(rowKeyBytes);
+ p.addColumn(FlowRunColumnFamily.INFO.getBytes(), columnNameBytes,
+ valueBytes);
+ Configuration hbaseConf = util.getConfiguration();
+ TableName table = TableName.valueOf(hbaseConf.get(
+ FlowRunTable.TABLE_NAME_CONF_NAME, FlowRunTable.DEFAULT_TABLE_NAME));
+ Connection conn = null;
+ conn = ConnectionFactory.createConnection(hbaseConf);
+ Table flowRunTable = conn.getTable(table);
+ flowRunTable.put(p);
+
+ Get g = new Get(rowKeyBytes);
+ Result r = flowRunTable.get(g);
+ assertNotNull(r);
+ assertTrue(r.size() >= 1);
+ Cell actualValue = r.getColumnLatestCell(
+ FlowRunColumnFamily.INFO.getBytes(), columnNameBytes);
+ assertNotNull(CellUtil.cloneValue(actualValue));
+ assertEquals(Bytes.toString(CellUtil.cloneValue(actualValue)), value);
+ }
+
@Test
public void testWriteFlowRunCompaction() throws Exception {
String cluster = "kompaction_cluster1";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-4356. Ensure the timeline
service v.2 is disabled cleanly and has no impact when it's turned off.
Contributed by Sangjin Lee.
Posted by gt...@apache.org.
YARN-4356. Ensure the timeline service v.2 is disabled cleanly and has no
impact when it's turned off. Contributed by Sangjin Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef71c1fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef71c1fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef71c1fc
Branch: refs/heads/YARN-2928
Commit: ef71c1fc4c37b355e895cea23a868ba57555d9b5
Parents: e987c7e
Author: Li Lu <gt...@apache.org>
Authored: Fri Dec 11 11:17:34 2015 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:01:41 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 63 ++++----
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 11 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 5 -
.../src/main/resources/mapred-default.xml | 7 -
.../mapred/TestMRTimelineEventHandling.java | 5 +-
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 58 ++++++-
.../distributedshell/ApplicationMaster.java | 153 ++++++++-----------
.../applications/distributedshell/Client.java | 16 --
.../distributedshell/TestDistributedShell.java | 10 +-
.../hadoop/yarn/client/api/TimelineClient.java | 18 ++-
.../client/api/impl/TimelineClientImpl.java | 3 +
.../src/main/resources/yarn-default.xml | 5 +-
.../impl/pb/NodeHeartbeatRequestPBImpl.java | 10 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 10 +-
.../hadoop/yarn/server/nodemanager/Context.java | 3 +-
.../yarn/server/nodemanager/NodeManager.java | 23 ++-
.../nodemanager/NodeStatusUpdaterImpl.java | 48 +++---
.../collectormanager/NMCollectorService.java | 10 +-
.../containermanager/ContainerManagerImpl.java | 62 +++++---
.../application/ApplicationImpl.java | 70 +++++++--
.../monitor/ContainersMonitorImpl.java | 11 +-
.../timelineservice/NMTimelinePublisher.java | 49 +++---
.../TestContainerManagerRecovery.java | 9 +-
.../application/TestApplication.java | 3 +-
.../nodemanager/webapp/TestNMWebServices.java | 8 +-
.../ApplicationMasterService.java | 11 +-
.../server/resourcemanager/ClientRMService.java | 35 +++--
.../server/resourcemanager/RMAppManager.java | 7 +-
.../server/resourcemanager/ResourceManager.java | 33 ++--
.../resourcemanager/ResourceTrackerService.java | 21 ++-
.../resourcemanager/amlauncher/AMLauncher.java | 15 +-
.../metrics/TimelineServiceV2Publisher.java | 2 +-
.../server/resourcemanager/rmapp/RMAppImpl.java | 16 +-
.../resourcemanager/TestClientRMService.java | 3 +
.../metrics/TestSystemMetricsPublisher.java | 2 +-
.../TestSystemMetricsPublisherForV2.java | 1 +
.../TestTimelineServiceClientIntegration.java | 30 +++-
.../PerNodeTimelineCollectorsAuxService.java | 15 +-
.../reader/TimelineReaderServer.java | 14 +-
...TestPerNodeTimelineCollectorsAuxService.java | 9 +-
.../reader/TestTimelineReaderServer.java | 3 +
.../reader/TestTimelineReaderWebServices.java | 2 +
...stTimelineReaderWebServicesHBaseStorage.java | 2 +
44 files changed, 520 insertions(+), 373 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index b1c1a52..d88588c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -19,9 +19,6 @@
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
import java.util.Collections;
import java.util.EnumSet;
import java.util.HashMap;
@@ -31,7 +28,11 @@ import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -57,9 +58,9 @@ import org.apache.hadoop.mapreduce.util.JobHistoryEventUtils;
import org.apache.hadoop.mapreduce.v2.api.records.JobId;
import org.apache.hadoop.mapreduce.v2.api.records.JobState;
import org.apache.hadoop.mapreduce.v2.app.AppContext;
+import org.apache.hadoop.mapreduce.v2.app.MRAppMaster;
import org.apache.hadoop.mapreduce.v2.app.job.Job;
import org.apache.hadoop.mapreduce.v2.app.job.JobStateInternal;
-import org.apache.hadoop.mapreduce.v2.app.MRAppMaster;
import org.apache.hadoop.mapreduce.v2.jobhistory.FileNameIndexUtils;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
@@ -81,11 +82,10 @@ import org.codehaus.jackson.node.ArrayNode;
import org.codehaus.jackson.node.JsonNodeFactory;
import org.codehaus.jackson.node.ObjectNode;
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-
import com.google.common.annotations.VisibleForTesting;
import com.sun.jersey.api.client.ClientHandlerException;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
* The job history events get routed to this class. This class writes the Job
* history events to the DFS directly into a staging dir and then moved to a
@@ -133,20 +133,17 @@ public class JobHistoryEventHandler extends AbstractService
protected static final Map<JobId, MetaInfo> fileMap =
Collections.<JobId,MetaInfo>synchronizedMap(new HashMap<JobId,MetaInfo>());
-
- // For posting entities in new timeline service in a non-blocking way
- // TODO YARN-3367 replace with event loop in TimelineClient.
- private static ExecutorService threadPool =
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
// should job completion be force when the AM shuts down?
protected volatile boolean forceJobCompletion = false;
protected TimelineClient timelineClient;
- private boolean newTimelineServiceEnabled = false;
+ private boolean timelineServiceV2Enabled = false;
+
+ // For posting entities in new timeline service in a non-blocking way
+ // TODO YARN-3367 replace with event loop in TimelineClient.
+ private ExecutorService threadPool;
private static String MAPREDUCE_JOB_ENTITY_TYPE = "MAPREDUCE_JOB";
private static String MAPREDUCE_TASK_ENTITY_TYPE = "MAPREDUCE_TASK";
@@ -276,22 +273,26 @@ public class JobHistoryEventHandler extends AbstractService
// configuration status: off, on_with_v1 or on_with_v2.
if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
-
+ LOG.info("Emitting job history data to the timeline service is enabled");
+ if (YarnConfiguration.timelineServiceEnabled(conf)) {
+
timelineClient =
((MRAppMaster.RunningAppContext)context).getTimelineClient();
timelineClient.init(conf);
- newTimelineServiceEnabled = conf.getBoolean(
- MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED,
- MRJobConfig.DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED);
- LOG.info("Timeline service is enabled: " + (newTimelineServiceEnabled? "v2" : "v1"));
- LOG.info("Emitting job history data to the timeline server is enabled");
+ timelineServiceV2Enabled =
+ YarnConfiguration.timelineServiceV2Enabled(conf);
+ LOG.info("Timeline service is enabled; version: " +
+ YarnConfiguration.getTimelineServiceVersion(conf));
+ if (timelineServiceV2Enabled) {
+ // initialize the thread pool for v.2 timeline service
+ threadPool = createThreadPool();
+ }
} else {
LOG.info("Timeline service is not enabled");
}
} else {
- LOG.info("Emitting job history data to the timeline server is not enabled");
+ LOG.info("Emitting job history data to the timeline server is not " +
+ "enabled");
}
// Flag for setting
@@ -459,19 +460,27 @@ public class JobHistoryEventHandler extends AbstractService
if (timelineClient != null) {
timelineClient.stop();
}
- shutdownAndAwaitTermination();
+ if (threadPool != null) {
+ shutdownAndAwaitTermination();
+ }
LOG.info("Stopped JobHistoryEventHandler. super.stop()");
super.serviceStop();
}
// TODO remove threadPool after adding non-blocking call in TimelineClient
- private static void shutdownAndAwaitTermination() {
+ private ExecutorService createThreadPool() {
+ return Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
+ }
+
+ private void shutdownAndAwaitTermination() {
threadPool.shutdown();
try {
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
threadPool.shutdownNow();
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
- LOG.error("ThreadPool did not terminate");
+ LOG.error("ThreadPool did not terminate");
}
} catch (InterruptedException ie) {
threadPool.shutdownNow();
@@ -633,7 +642,7 @@ public class JobHistoryEventHandler extends AbstractService
processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(),
event.getJobID());
if (timelineClient != null) {
- if (newTimelineServiceEnabled) {
+ if (timelineServiceV2Enabled) {
processEventForNewTimelineService(historyEvent, event.getJobID(),
event.getTimestamp());
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 3279d03..897e2aa 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1058,14 +1058,9 @@ public class MRAppMaster extends CompositeService {
this.taskAttemptFinishingMonitor = taskAttemptFinishingMonitor;
if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)
- && conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
-
- boolean newTimelineServiceEnabled = conf.getBoolean(
- MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED,
- MRJobConfig.DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED);
-
- if (newTimelineServiceEnabled) {
+ && YarnConfiguration.timelineServiceEnabled(conf)) {
+
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
// create new version TimelineClient
timelineClient = TimelineClient.createTimelineClient(
appAttemptID.getApplicationId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index bc8aeda..c98746a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -467,11 +467,6 @@ public interface MRJobConfig {
"mapreduce.job.emit-timeline-data";
public static final boolean DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA =
false;
-
- public static final String MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED =
- "mapreduce.job.new-timeline-service.enabled";
- public static final boolean DEFAULT_MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED =
- false;
public static final String MR_PREFIX = "yarn.app.mapreduce.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index df3bd45..b7bdcc8 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -628,13 +628,6 @@
</description>
</property>
- <property>
- <name>mapreduce.job.new-timeline-service.enabled</name>
- <value>false</value>
- <description>Specifies if posting job and task events to new timeline service.
- </description>
-</property>
-
<property>
<name>mapreduce.input.fileinputformat.split.minsize</name>
<value>0</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index afc4686..1896c7b 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -166,11 +166,10 @@ public class TestMRTimelineEventHandling {
LOG.info("testMRNewTimelineServiceEventHandling start.");
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ // enable new timeline service
+ conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 2.0f);
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
- // enable new timeline serivce in MR side
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_NEW_TIMELINE_SERVICE_ENABLED, true);
-
// enable aux-service based timeline collectors
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 18a4c14..edb825d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -173,7 +173,7 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
boolean enableTimelineAuxService = false;
if (nmAuxServices != null) {
for (String nmAuxService: nmAuxServices) {
- if (nmAuxService == TIMELINE_AUX_SERVICE_NAME) {
+ if (nmAuxService.equals(TIMELINE_AUX_SERVICE_NAME)) {
enableTimelineAuxService = true;
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8892cd4..ed81eaa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -83,6 +83,10 @@ public class YarnConfiguration extends Configuration {
new DeprecationDelta("yarn.client.max-nodemanagers-proxies",
NM_CLIENT_MAX_NM_PROXIES)
});
+ Configuration.addDeprecations(new DeprecationDelta[] {
+ new DeprecationDelta(RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
+ SYSTEM_METRICS_PUBLISHER_ENABLED)
+ });
}
//Configurations
@@ -456,7 +460,8 @@ public class YarnConfiguration extends Configuration {
/**
* The setting that controls whether yarn system metrics is published on the
- * timeline server or not by RM. This configuration setting is for ATS V1
+ * timeline server or not by RM. This configuration setting is for ATS V1.
+ * This is now deprecated in favor of SYSTEM_METRICS_PUBLISHER_ENABLED.
*/
public static final String RM_SYSTEM_METRICS_PUBLISHER_ENABLED = RM_PREFIX
+ "system-metrics-publisher.enabled";
@@ -2675,13 +2680,52 @@ public class YarnConfiguration extends Configuration {
}
return clusterId;
}
-
- public static boolean systemMetricsPublisherEnabled(Configuration conf) {
+
+ // helper methods for timeline service configuration
+ /**
+ * Returns whether the timeline service is enabled via configuration.
+ *
+ * @param conf the configuration
+ * @return whether the timeline service is enabled.
+ */
+ public static boolean timelineServiceEnabled(Configuration conf) {
return conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)
- && conf.getBoolean(
- YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
- YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED);
+ }
+
+ /**
+ * Returns the timeline service version. It does not check whether the
+ * timeline service itself is enabled.
+ *
+ * @param conf the configuration
+ * @return the timeline service version as a float.
+ */
+ public static float getTimelineServiceVersion(Configuration conf) {
+ return conf.getFloat(TIMELINE_SERVICE_VERSION,
+ DEFAULT_TIMELINE_SERVICE_VERSION);
+ }
+
+ /**
+ * Returns whether the timeline service v.2 is enabled via configuration.
+ *
+ * @param conf the configuration
+ * @return whether the timeline service v.2 is enabled. V.2 refers to a
+ * version greater than equal to 2 but smaller than 3.
+ */
+ public static boolean timelineServiceV2Enabled(Configuration conf) {
+ return timelineServiceEnabled(conf) &&
+ (int)getTimelineServiceVersion(conf) == 2;
+ }
+
+ /**
+ * Returns whether the system publisher is enabled.
+ *
+ * @param conf the configuration
+ * @return whether the system publisher is enabled.
+ */
+ public static boolean systemMetricsPublisherEnabled(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED,
+ YarnConfiguration.DEFAULT_SYSTEM_METRICS_PUBLISHER_ENABLED);
}
/* For debugging. mp configurations to system output as XML format. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 1a9a4ca..6bd3c26 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -223,14 +223,11 @@ public class ApplicationMaster {
// Tracking url to which app master publishes info for clients to monitor
private String appMasterTrackingUrl = "";
- private boolean newTimelineService = false;
+ private boolean timelineServiceV2 = false;
// For posting entities in new timeline service in a non-blocking way
// TODO replace with event loop in TimelineClient.
- private static ExecutorService threadPool =
- Executors.newCachedThreadPool(
- new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
- .build());
+ private ExecutorService threadPool;
// App Master configuration
// No. of containers to run shell command on
@@ -331,8 +328,10 @@ public class ApplicationMaster {
}
appMaster.run();
result = appMaster.finish();
-
- shutdownAndAwaitTermination();
+
+ if (appMaster.threadPool != null) {
+ appMaster.shutdownAndAwaitTermination();
+ }
} catch (Throwable t) {
LOG.fatal("Error running ApplicationMaster", t);
LogManager.shutdown();
@@ -346,16 +345,22 @@ public class ApplicationMaster {
System.exit(2);
}
}
-
+
//TODO remove threadPool after adding non-blocking call in TimelineClient
- private static void shutdownAndAwaitTermination() {
+ private ExecutorService createThreadPool() {
+ return Executors.newCachedThreadPool(
+ new ThreadFactoryBuilder().setNameFormat("TimelineService #%d")
+ .build());
+ }
+
+ private void shutdownAndAwaitTermination() {
threadPool.shutdown();
try {
// Wait a while for existing tasks to terminate
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS)) {
threadPool.shutdownNow();
if (!threadPool.awaitTermination(60, TimeUnit.SECONDS))
- LOG.error("ThreadPool did not terminate");
+ LOG.error("ThreadPool did not terminate");
}
} catch (InterruptedException ie) {
threadPool.shutdownNow();
@@ -433,8 +438,7 @@ public class ApplicationMaster {
opts.addOption("container_retry_interval", true,
"Interval between each retry, unit is milliseconds");
opts.addOption("debug", false, "Dump out debug information");
- opts.addOption("timeline_service_version", true,
- "Version for timeline service");
+
opts.addOption("help", false, "Print usage");
CommandLine cliParser = new GnuParser().parse(opts, args);
@@ -586,27 +590,15 @@ public class ApplicationMaster {
containrRetryInterval = Integer.parseInt(cliParser.getOptionValue(
"container_retry_interval", "0"));
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
- if (cliParser.hasOption("timeline_service_version")) {
- String timelineServiceVersion =
- cliParser.getOptionValue("timeline_service_version", "v1");
- if (timelineServiceVersion.trim().equalsIgnoreCase("v1")) {
- newTimelineService = false;
- } else if (timelineServiceVersion.trim().equalsIgnoreCase("v2")) {
- newTimelineService = true;
- } else {
- throw new IllegalArgumentException(
- "timeline_service_version is not set properly, should be 'v1' or 'v2'");
- }
+ if (YarnConfiguration.timelineServiceEnabled(conf)) {
+ timelineServiceV2 =
+ YarnConfiguration.timelineServiceV2Enabled(conf);
+ if (timelineServiceV2) {
+ threadPool = createThreadPool();
}
} else {
timelineClient = null;
LOG.warn("Timeline service is not enabled");
- if (cliParser.hasOption("timeline_service_version")) {
- throw new IllegalArgumentException(
- "Timeline service is not enabled");
- }
}
return true;
@@ -668,16 +660,17 @@ public class ApplicationMaster {
nmClientAsync.start();
startTimelineClient(conf);
- // need to bind timelineClient
- amRMClient.registerTimelineClient(timelineClient);
+ if (timelineServiceV2) {
+ // need to bind timelineClient
+ amRMClient.registerTimelineClient(timelineClient);
+ }
if(timelineClient != null) {
- if (newTimelineService) {
- publishApplicationAttemptEventOnNewTimelineService(timelineClient,
- appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_START, domainId,
- appSubmitterUgi);
+ if (timelineServiceV2) {
+ publishApplicationAttemptEventOnTimelineServiceV2(
+ DSEvent.DS_APP_ATTEMPT_START);
} else {
publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
- DSEvent.DS_APP_ATTEMPT_START, domainId, appSubmitterUgi);
+ DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
}
}
@@ -748,10 +741,9 @@ public class ApplicationMaster {
appSubmitterUgi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
public Void run() throws Exception {
- if (conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+ if (YarnConfiguration.timelineServiceEnabled(conf)) {
// Creating the Timeline Client
- if (newTimelineService) {
+ if (timelineServiceV2) {
timelineClient = TimelineClient.createTimelineClient(
appAttemptID.getApplicationId());
} else {
@@ -787,10 +779,9 @@ public class ApplicationMaster {
}
if (timelineClient != null) {
- if (newTimelineService) {
- publishApplicationAttemptEventOnNewTimelineService(timelineClient,
- appAttemptID.toString(), DSEvent.DS_APP_ATTEMPT_END, domainId,
- appSubmitterUgi);
+ if (timelineServiceV2) {
+ publishApplicationAttemptEventOnTimelineServiceV2(
+ DSEvent.DS_APP_ATTEMPT_END);
} else {
publishApplicationAttemptEvent(timelineClient, appAttemptID.toString(),
DSEvent.DS_APP_ATTEMPT_END, domainId, appSubmitterUgi);
@@ -900,12 +891,11 @@ public class ApplicationMaster {
+ containerStatus.getContainerId());
}
if(timelineClient != null) {
- if (newTimelineService) {
- publishContainerEndEventOnNewTimelineService(
- timelineClient, containerStatus, domainId, appSubmitterUgi);
+ if (timelineServiceV2) {
+ publishContainerEndEventOnTimelineServiceV2(containerStatus);
} else {
publishContainerEndEvent(
- timelineClient, containerStatus, domainId, appSubmitterUgi);
+ timelineClient, containerStatus, domainId, appSubmitterUgi);
}
}
}
@@ -1032,14 +1022,13 @@ public class ApplicationMaster {
applicationMaster.timelineClient, container,
applicationMaster.domainId, applicationMaster.appSubmitterUgi);
- if (applicationMaster.newTimelineService) {
- ApplicationMaster.publishContainerStartEventOnNewTimelineService(
- applicationMaster.timelineClient, container,
- applicationMaster.domainId, applicationMaster.appSubmitterUgi);
+ if (applicationMaster.timelineServiceV2) {
+ applicationMaster.publishContainerStartEventOnTimelineServiceV2(
+ container);
} else {
applicationMaster.publishContainerStartEvent(
- applicationMaster.timelineClient, container,
- applicationMaster.domainId, applicationMaster.appSubmitterUgi);
+ applicationMaster.timelineClient, container,
+ applicationMaster.domainId, applicationMaster.appSubmitterUgi);
}
}
}
@@ -1349,7 +1338,7 @@ public class ApplicationMaster {
LOG.error("App Attempt "
+ (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
+ " event could not be published for "
- + appAttemptId.toString(), e);
+ + appAttemptID, e);
}
}
@@ -1397,27 +1386,24 @@ public class ApplicationMaster {
return new Thread(runnableLaunchContainer);
}
- private static void publishContainerStartEventOnNewTimelineService(
- final TimelineClient timelineClient, final Container container,
- final String domainId, final UserGroupInformation ugi) {
+ private void publishContainerStartEventOnTimelineServiceV2(
+ final Container container) {
Runnable publishWrapper = new Runnable() {
public void run() {
- publishContainerStartEventOnNewTimelineServiceBase(timelineClient,
- container, domainId, ugi);
+ publishContainerStartEventOnTimelineServiceV2Base(container);
}
};
threadPool.execute(publishWrapper);
}
- private static void publishContainerStartEventOnNewTimelineServiceBase(
- final TimelineClient timelineClient, Container container, String domainId,
- UserGroupInformation ugi) {
+ private void publishContainerStartEventOnTimelineServiceV2Base(
+ Container container) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId);
- entity.addInfo("user", ugi.getShortUserName());
+ entity.addInfo("user", appSubmitterUgi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
@@ -1428,7 +1414,7 @@ public class ApplicationMaster {
entity.addEvent(event);
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ appSubmitterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public TimelinePutResponse run() throws Exception {
timelineClient.putEntities(entity);
@@ -1442,27 +1428,24 @@ public class ApplicationMaster {
}
}
- private static void publishContainerEndEventOnNewTimelineService(
- final TimelineClient timelineClient, final ContainerStatus container,
- final String domainId, final UserGroupInformation ugi) {
+ private void publishContainerEndEventOnTimelineServiceV2(
+ final ContainerStatus container) {
Runnable publishWrapper = new Runnable() {
public void run() {
- publishContainerEndEventOnNewTimelineServiceBase(timelineClient,
- container, domainId, ugi);
+ publishContainerEndEventOnTimelineServiceV2Base(container);
}
};
threadPool.execute(publishWrapper);
}
- private static void publishContainerEndEventOnNewTimelineServiceBase(
- final TimelineClient timelineClient, final ContainerStatus container,
- final String domainId, final UserGroupInformation ugi) {
+ private void publishContainerEndEventOnTimelineServiceV2Base(
+ final ContainerStatus container) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
entity.setId(container.getContainerId().toString());
entity.setType(DSEntity.DS_CONTAINER.toString());
//entity.setDomainId(domainId);
- entity.addInfo("user", ugi.getShortUserName());
+ entity.addInfo("user", appSubmitterUgi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setTimestamp(System.currentTimeMillis());
@@ -1472,7 +1455,7 @@ public class ApplicationMaster {
entity.addEvent(event);
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ appSubmitterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public TimelinePutResponse run() throws Exception {
timelineClient.putEntities(entity);
@@ -1486,29 +1469,25 @@ public class ApplicationMaster {
}
}
- private static void publishApplicationAttemptEventOnNewTimelineService(
- final TimelineClient timelineClient, final String appAttemptId,
- final DSEvent appEvent, final String domainId,
- final UserGroupInformation ugi) {
+ private void publishApplicationAttemptEventOnTimelineServiceV2(
+ final DSEvent appEvent) {
Runnable publishWrapper = new Runnable() {
public void run() {
- publishApplicationAttemptEventOnNewTimelineServiceBase(timelineClient,
- appAttemptId, appEvent, domainId, ugi);
+ publishApplicationAttemptEventOnTimelineServiceV2Base(appEvent);
}
};
threadPool.execute(publishWrapper);
}
- private static void publishApplicationAttemptEventOnNewTimelineServiceBase(
- final TimelineClient timelineClient, String appAttemptId,
- DSEvent appEvent, String domainId, UserGroupInformation ugi) {
+ private void publishApplicationAttemptEventOnTimelineServiceV2Base(
+ DSEvent appEvent) {
final org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
- entity.setId(appAttemptId);
+ entity.setId(appAttemptID.toString());
entity.setType(DSEntity.DS_APP_ATTEMPT.toString());
//entity.setDomainId(domainId);
- entity.addInfo("user", ugi.getShortUserName());
+ entity.addInfo("user", appSubmitterUgi.getShortUserName());
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent event =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent();
event.setId(appEvent.toString());
@@ -1516,7 +1495,7 @@ public class ApplicationMaster {
entity.addEvent(event);
try {
- ugi.doAs(new PrivilegedExceptionAction<Object>() {
+ appSubmitterUgi.doAs(new PrivilegedExceptionAction<Object>() {
@Override
public TimelinePutResponse run() throws Exception {
timelineClient.putEntities(entity);
@@ -1527,7 +1506,7 @@ public class ApplicationMaster {
LOG.error("App Attempt "
+ (appEvent.equals(DSEvent.DS_APP_ATTEMPT_START) ? "start" : "end")
+ " event could not be published for "
- + appAttemptId.toString(),
+ + appAttemptID,
e instanceof UndeclaredThrowableException ? e.getCause() : e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
index 43ffa09..4ae7edd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
@@ -195,8 +195,6 @@ public class Client {
// Command line options
private Options opts;
- private String timelineServiceVersion;
-
private static final String shellCommandPath = "shellCommands";
private static final String shellArgsPath = "shellArgs";
private static final String appMasterJarPath = "AppMaster.jar";
@@ -272,7 +270,6 @@ public class Client {
opts.addOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command");
opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed");
opts.addOption("log_properties", true, "log4j.properties file");
- opts.addOption("timeline_service_version", true, "Version for timeline service");
opts.addOption("keep_containers_across_application_attempts", false,
"Flag to indicate whether to keep containers across application attempts." +
" If the flag is true, running containers will not be killed when" +
@@ -386,16 +383,6 @@ public class Client {
+ " Specified virtual cores=" + amVCores);
}
- if (cliParser.hasOption("timeline_service_version")) {
- timelineServiceVersion =
- cliParser.getOptionValue("timeline_service_version", "v1");
- if (! (timelineServiceVersion.trim().equalsIgnoreCase("v1") ||
- timelineServiceVersion.trim().equalsIgnoreCase("v2"))) {
- throw new IllegalArgumentException(
- "timeline_service_version is not set properly, should be 'v1' or 'v2'");
- }
- }
-
if (!cliParser.hasOption("jar")) {
throw new IllegalArgumentException("No jar file specified for application master");
}
@@ -725,9 +712,6 @@ public class Client {
vargs.addAll(containerRetryOptions);
- if (timelineServiceVersion != null) {
- vargs.add("--timeline_service_version " + timelineServiceVersion);
- }
vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 80ffd15..bdf6e2b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -138,9 +138,10 @@ public class TestDistributedShell {
conf.set("yarn.log.dir", "target");
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
// mark if we need to launch the v1 timeline server
- boolean enableATSServer = true;
// disable aux-service based timeline aggregators
conf.set(YarnConfiguration.NM_AUX_SERVICES, "");
+ conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, true);
+
conf.set(YarnConfiguration.NM_VMEM_PMEM_RATIO, "8");
conf.set(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class.getName());
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
@@ -157,7 +158,6 @@ public class TestDistributedShell {
true);
conf.setBoolean(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED,
true);
- conf.setBoolean(YarnConfiguration.SYSTEM_METRICS_PUBLISHER_ENABLED, false);
// ATS version specific settings
if (timelineVersion == 1.0f) {
@@ -177,7 +177,6 @@ public class TestDistributedShell {
DistributedShellTimelinePlugin.class.getName());
} else if (timelineVersion == 2.0f) {
// disable v1 timeline server since we no longer have a server here
- enableATSServer = false;
// enable aux-service based timeline aggregators
conf.set(YarnConfiguration.NM_AUX_SERVICES, TIMELINE_AUX_SERVICE_NAME);
conf.set(YarnConfiguration.NM_AUX_SERVICES + "." + TIMELINE_AUX_SERVICE_NAME
@@ -331,12 +330,7 @@ public class TestDistributedShell {
}
boolean isTestingTimelineV2 = false;
if (timelineVersionWatcher.getTimelineVersion() == 2.0f) {
- String[] timelineArgs = {
- "--timeline_service_version",
- "v2"
- };
isTestingTimelineV2 = true;
- args = mergeArgs(args, timelineArgs);
if (!defaultFlow) {
String[] flowArgs = {
"--flow_name",
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
index d629bb7..e0f2267 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/TimelineClient.java
@@ -54,17 +54,21 @@ public abstract class TimelineClient extends AbstractService implements
* current user may use {@link UserGroupInformation#doAs} another user to
* construct and initialize a timeline client if the following operations are
* supposed to be conducted by that user.
- *
- * @return a timeline client
*/
protected ApplicationId contextAppId;
+ /**
+ * Creates an instance of the timeline v.1.x client.
+ */
@Public
public static TimelineClient createTimelineClient() {
TimelineClient client = new TimelineClientImpl();
return client;
}
+ /**
+ * Creates an instance of the timeline v.2 client.
+ */
@Public
public static TimelineClient createTimelineClient(ApplicationId appId) {
TimelineClient client = new TimelineClientImpl(appId);
@@ -203,8 +207,9 @@ public abstract class TimelineClient extends AbstractService implements
/**
* <p>
* Send the information of a number of conceptual entities to the timeline
- * aggregator. It is a blocking API. The method will not return until all the
- * put entities have been persisted.
+ * service v.2 collector. It is a blocking API. The method will not return
+ * until all the put entities have been persisted. If this method is invoked
+ * for a non-v.2 timeline client instance, a YarnException is thrown.
* </p>
*
* @param entities
@@ -220,8 +225,9 @@ public abstract class TimelineClient extends AbstractService implements
/**
* <p>
* Send the information of a number of conceptual entities to the timeline
- * aggregator. It is an asynchronous API. The method will return once all the
- * entities are received.
+ * service v.2 collector. It is an asynchronous API. The method will return
+ * once all the entities are received. If this method is invoked for a
+ * non-v.2 timeline client instance, a YarnException is thrown.
* </p>
*
* @param entities
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 0096879..f92e6a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -388,6 +388,9 @@ public class TimelineClientImpl extends TimelineClient {
private void putEntities(boolean async,
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity... entities)
throws IOException, YarnException {
+ if (!timelineServiceV2) {
+ throw new YarnException("v.2 method is invoked on a v.1.x client");
+ }
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities
entitiesContainer =
new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntities();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index cc81402..e4b562e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -785,14 +785,15 @@
<property>
<description>The setting that controls whether yarn system metrics is
published to the Timeline server (version one) or not, by RM.
- This configuration is deprecated.</description>
+ This configuration is now deprecated in favor of
+ yarn.system-metrics-publisher.enabled.</description>
<name>yarn.resourcemanager.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
<property>
<description>The setting that controls whether yarn system metrics is
- published on the Timeline server (version two) or not by RM And NM.</description>
+ published on the Timeline service or not by RM And NM.</description>
<name>yarn.system-metrics-publisher.enabled</name>
<value>false</value>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
index fa0cf5c..066abfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatRequestPBImpl.java
@@ -261,10 +261,12 @@ public class NodeHeartbeatRequestPBImpl extends NodeHeartbeatRequest {
private void initRegisteredCollectors() {
NodeHeartbeatRequestProtoOrBuilder p = viaProto ? proto : builder;
List<AppCollectorsMapProto> list = p.getRegisteredCollectorsList();
- this.registeredCollectors = new HashMap<ApplicationId, String> ();
- for (AppCollectorsMapProto c : list) {
- ApplicationId appId = convertFromProtoFormat(c.getAppId());
- this.registeredCollectors.put(appId, c.getAppCollectorAddr());
+ if (!list.isEmpty()) {
+ this.registeredCollectors = new HashMap<>();
+ for (AppCollectorsMapProto c : list) {
+ ApplicationId appId = convertFromProtoFormat(c.getAppId());
+ this.registeredCollectors.put(appId, c.getAppCollectorAddr());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 2521b9c..151006b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -531,10 +531,12 @@ public class NodeHeartbeatResponsePBImpl extends
private void initAppCollectorsMap() {
NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
List<AppCollectorsMapProto> list = p.getAppCollectorsMapList();
- this.appCollectorsMap = new HashMap<ApplicationId, String> ();
- for (AppCollectorsMapProto c : list) {
- ApplicationId appId = convertFromProtoFormat(c.getAppId());
- this.appCollectorsMap.put(appId, c.getAppCollectorAddr());
+ if (!list.isEmpty()) {
+ this.appCollectorsMap = new HashMap<>();
+ for (AppCollectorsMapProto c : list) {
+ ApplicationId appId = convertFromProtoFormat(c.getAppId());
+ this.appCollectorsMap.put(appId, c.getAppCollectorAddr());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 4c2245e..9305d25 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -75,7 +75,8 @@ public interface Context {
/**
* Get the registered collectors that located on this NM.
- * @return registered
+ * @return registered collectors, or null if the timeline service v.2 is not
+ * enabled
*/
Map<ApplicationId, String> getRegisteredCollectors();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 50e122c..b41a81a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -100,6 +100,7 @@ public class NodeManager extends CompositeService
private Context context;
private AsyncDispatcher dispatcher;
private ContainerManagerImpl containerManager;
+ // the NM collector service is set only if the timeline service v.2 is enabled
private NMCollectorService nmCollectorService;
private NodeStatusUpdater nodeStatusUpdater;
private NodeResourceMonitor nodeResourceMonitor;
@@ -375,8 +376,10 @@ public class NodeManager extends CompositeService
DefaultMetricsSystem.initialize("NodeManager");
- this.nmCollectorService = createNMCollectorService(context);
- addService(nmCollectorService);
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ this.nmCollectorService = createNMCollectorService(context);
+ addService(nmCollectorService);
+ }
// StatusUpdater should be added last so that it get started last
// so that we make sure everything is up before registering with RM.
@@ -473,8 +476,7 @@ public class NodeManager extends CompositeService
protected final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
- protected Map<ApplicationId, String> registeredCollectors =
- new ConcurrentHashMap<ApplicationId, String>();
+ protected Map<ApplicationId, String> registeredCollectors;
protected final ConcurrentMap<ContainerId,
org.apache.hadoop.yarn.api.records.Container> increasedContainers =
@@ -507,6 +509,9 @@ public class NodeManager extends CompositeService
LocalDirsHandlerService dirsHandler, ApplicationACLsManager aclsManager,
NMStateStoreService stateStore, boolean isDistSchedulingEnabled,
Configuration conf) {
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ this.registeredCollectors = new ConcurrentHashMap<>();
+ }
this.containerTokenSecretManager = containerTokenSecretManager;
this.nmTokenSecretManager = nmTokenSecretManager;
this.dirsHandler = dirsHandler;
@@ -783,7 +788,14 @@ public class NodeManager extends CompositeService
return this.context;
}
- // For testing
+ /**
+ * Returns the NM collector service. It should be used only for testing
+ * purposes.
+ *
+ * @return the NM collector service, or null if the timeline service v.2 is
+ * not enabled
+ */
+ @VisibleForTesting
NMCollectorService getNMCollectorService() {
return this.nmCollectorService;
}
@@ -791,6 +803,7 @@ public class NodeManager extends CompositeService
public static void main(String[] args) throws IOException {
Thread.setDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler());
StringUtils.startupShutdownMessage(NodeManager.class, args, LOG);
+ @SuppressWarnings("resource")
NodeManager nodeManager = new NodeManager();
Configuration conf = new YarnConfiguration();
new GenericOptionsParser(conf, args);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 08334ae..34dfdd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -840,7 +840,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
dispatcher.getEventHandler().handle(
new CMgrSignalContainersEvent(containersToSignal));
}
- if (YarnConfiguration.systemMetricsPublisherEnabled(context.getConf())) {
+ if (YarnConfiguration.timelineServiceV2Enabled(context.getConf())) {
updateTimelineClientsAddress(response);
}
@@ -874,7 +874,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
/**
* Caller should take care of sending non null nodelabels for both
* arguments
- *
+ *
* @param nodeLabelsNew
* @param nodeLabelsOld
* @return if the New node labels are diff from the older one.
@@ -890,27 +890,37 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
private void updateTimelineClientsAddress(
NodeHeartbeatResponse response) {
- Set<Map.Entry<ApplicationId, String>> rmKnownCollectors =
- response.getAppCollectorsMap().entrySet();
- for (Map.Entry<ApplicationId, String> entry : rmKnownCollectors) {
- ApplicationId appId = entry.getKey();
- String collectorAddr = entry.getValue();
-
- // Only handle applications running on local node.
- // Not include apps with timeline collectors running in local
- Application application = context.getApplications().get(appId);
- if (application != null &&
- !context.getRegisteredCollectors().containsKey(appId)) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Sync a new collector address: " + collectorAddr +
- " for application: " + appId + " from RM.");
+ Map<ApplicationId, String> knownCollectorsMap =
+ response.getAppCollectorsMap();
+ if (knownCollectorsMap == null) {
+ LOG.warn("the collectors map is null");
+ } else {
+ Set<Map.Entry<ApplicationId, String>> rmKnownCollectors =
+ knownCollectorsMap.entrySet();
+ for (Map.Entry<ApplicationId, String> entry : rmKnownCollectors) {
+ ApplicationId appId = entry.getKey();
+ String collectorAddr = entry.getValue();
+
+ // Only handle applications running on local node.
+ // Not include apps with timeline collectors running in local
+ Application application = context.getApplications().get(appId);
+ // TODO this logic could be problematic if the collector address
+ // gets updated due to NM restart or collector service failure
+ if (application != null &&
+ !context.getRegisteredCollectors().containsKey(appId)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Sync a new collector address: " + collectorAddr +
+ " for application: " + appId + " from RM.");
+ }
+ TimelineClient client = application.getTimelineClient();
+ if (client != null) {
+ client.setTimelineServiceAddress(collectorAddr);
+ }
}
- TimelineClient client = application.getTimelineClient();
- client.setTimelineServiceAddress(collectorAddr);
}
}
}
-
+
private void updateMasterKeys(NodeHeartbeatResponse response) {
// See if the master-key has rolled over
MasterKey updatedMasterKey = response.getContainerTokenMasterKey();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index db79ee5..3ba81ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -43,6 +43,10 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+/**
+ * Service that handles collector information. It is used only if the timeline
+ * service v.2 is enabled.
+ */
public class NMCollectorService extends CompositeService implements
CollectorNodemanagerProtocol {
@@ -113,9 +117,9 @@ public class NMCollectorService extends CompositeService implements
String collectorAddr = collector.getCollectorAddr();
newCollectorsMap.put(appId, collectorAddr);
// set registered collector address to TimelineClient.
- if (YarnConfiguration.systemMetricsPublisherEnabled(context.getConf())) {
- TimelineClient client =
- context.getApplications().get(appId).getTimelineClient();
+ TimelineClient client =
+ context.getApplications().get(appId).getTimelineClient();
+ if (client != null) {
client.setTimelineServiceAddress(collectorAddr);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 33aeac4..46571f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -115,6 +115,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationFinishEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationImpl.FlowContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationInitEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
@@ -190,7 +191,8 @@ public class ContainerManagerImpl extends CompositeService implements
private long waitForContainersOnShutdownMillis;
- private final NMTimelinePublisher nmMetricsPublisher;
+ // NM metrics publisher is set only if the timeline service v.2 is enabled
+ private NMTimelinePublisher nmMetricsPublisher;
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
@@ -218,9 +220,17 @@ public class ContainerManagerImpl extends CompositeService implements
auxiliaryServices.registerServiceListener(this);
addService(auxiliaryServices);
- nmMetricsPublisher = createNMTimelinePublisher(context);
- context.setNMTimelinePublisher(nmMetricsPublisher);
- this.containersMonitor = createContainersMonitor(exec);
+ // initialize the metrics publisher if the timeline service v.2 is enabled
+ // and the system publisher is enabled
+ Configuration conf = context.getConf();
+ if (YarnConfiguration.timelineServiceV2Enabled(conf) &&
+ YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ LOG.info("YARN system metrics publishing service is enabled");
+ nmMetricsPublisher = createNMTimelinePublisher(context);
+ context.setNMTimelinePublisher(nmMetricsPublisher);
+ }
+ this.containersMonitor =
+ new ContainersMonitorImpl(exec, dispatcher, this.context);
addService(this.containersMonitor);
dispatcher.register(ContainerEventType.class,
@@ -236,7 +246,6 @@ public class ContainerManagerImpl extends CompositeService implements
addService(dispatcher);
-
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
@@ -341,7 +350,7 @@ public class ContainerManagerImpl extends CompositeService implements
LOG.info("Recovering application " + appId);
//TODO: Recover flow and flow run ID
ApplicationImpl app = new ApplicationImpl(
- dispatcher, p.getUser(), null, null, 0, appId, creds, context);
+ dispatcher, p.getUser(), appId, creds, context);
context.getApplications().put(appId, app);
app.handle(new ApplicationInitEvent(appId, acls, logAggregationContext));
}
@@ -959,20 +968,27 @@ public class ContainerManagerImpl extends CompositeService implements
try {
if (!isServiceStopped()) {
// Create the application
- String flowName = launchContext.getEnvironment().get(
- TimelineUtils.FLOW_NAME_TAG_PREFIX);
- String flowVersion = launchContext.getEnvironment().get(
- TimelineUtils.FLOW_VERSION_TAG_PREFIX);
- String flowRunIdStr = launchContext.getEnvironment().get(
- TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
- long flowRunId = 0L;
- if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
- flowRunId = Long.parseLong(flowRunIdStr);
+ // populate the flow context from the launch context if the timeline
+ // service v.2 is enabled
+ FlowContext flowContext = null;
+ if (YarnConfiguration.timelineServiceV2Enabled(getConfig())) {
+ String flowName = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_NAME_TAG_PREFIX);
+ String flowVersion = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_VERSION_TAG_PREFIX);
+ String flowRunIdStr = launchContext.getEnvironment().get(
+ TimelineUtils.FLOW_RUN_ID_TAG_PREFIX);
+ long flowRunId = 0L;
+ if (flowRunIdStr != null && !flowRunIdStr.isEmpty()) {
+ flowRunId = Long.parseLong(flowRunIdStr);
+ }
+ flowContext =
+ new FlowContext(flowName, flowVersion, flowRunId);
}
if (!context.getApplications().containsKey(applicationID)) {
Application application =
- new ApplicationImpl(dispatcher, user, flowName, flowVersion,
- flowRunId, applicationID, credentials, context);
+ new ApplicationImpl(dispatcher, user, flowContext,
+ applicationID, credentials, context);
if (context.getApplications().putIfAbsent(applicationID,
application) == null) {
LOG.info("Creating a new application reference for app "
@@ -1324,7 +1340,9 @@ public class ContainerManagerImpl extends CompositeService implements
Container c = containers.get(event.getContainerID());
if (c != null) {
c.handle(event);
- nmMetricsPublisher.publishContainerEvent(event);
+ if (nmMetricsPublisher != null) {
+ nmMetricsPublisher.publishContainerEvent(event);
+ }
} else {
LOG.warn("Event " + event + " sent to absent container " +
event.getContainerID());
@@ -1340,7 +1358,9 @@ public class ContainerManagerImpl extends CompositeService implements
event.getApplicationID());
if (app != null) {
app.handle(event);
- nmMetricsPublisher.publishApplicationEvent(event);
+ if (nmMetricsPublisher != null) {
+ nmMetricsPublisher.publishApplicationEvent(event);
+ }
} else {
LOG.warn("Event " + event + " sent to absent application "
+ event.getApplicationID());
@@ -1363,7 +1383,9 @@ public class ContainerManagerImpl extends CompositeService implements
@Override
public void handle(LocalizationEvent event) {
origLocalizationEventHandler.handle(event);
- timelinePublisher.publishLocalizationEvent(event);
+ if (timelinePublisher != null) {
+ timelinePublisher.publishLocalizationEvent(event);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 6e87cfd..93c6758 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -67,9 +67,8 @@ public class ApplicationImpl implements Application {
final Dispatcher dispatcher;
final String user;
- final String flowName;
- final String flowVersion;
- final long flowRunId;
+ // flow context is set only if the timeline service v.2 is enabled
+ private FlowContext flowContext;
final ApplicationId appId;
final Credentials credentials;
Map<ApplicationAccessType, String> applicationACLs;
@@ -86,14 +85,16 @@ public class ApplicationImpl implements Application {
Map<ContainerId, Container> containers =
new HashMap<ContainerId, Container>();
- public ApplicationImpl(Dispatcher dispatcher, String user, String flowName,
- String flowVersion, long flowRunId, ApplicationId appId,
- Credentials credentials, Context context) {
+ public ApplicationImpl(Dispatcher dispatcher, String user,
+ ApplicationId appId, Credentials credentials, Context context) {
+ this(dispatcher, user, null, appId, credentials, context);
+ }
+
+ public ApplicationImpl(Dispatcher dispatcher, String user,
+ FlowContext flowContext, ApplicationId appId, Credentials credentials,
+ Context context) {
this.dispatcher = dispatcher;
this.user = user;
- this.flowName = flowName;
- this.flowVersion = flowVersion;
- this.flowRunId = flowRunId;
this.appId = appId;
this.credentials = credentials;
this.aclsManager = context.getApplicationACLsManager();
@@ -103,11 +104,44 @@ public class ApplicationImpl implements Application {
writeLock = lock.writeLock();
stateMachine = stateMachineFactory.make(this);
Configuration conf = context.getConf();
- if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- createAndStartTimelineClient(conf);
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ if (flowContext == null) {
+ throw new IllegalArgumentException("flow context cannot be null");
+ }
+ this.flowContext = flowContext;
+ if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
+ createAndStartTimelineClient(conf);
+ }
}
}
-
+
+ /**
+ * Data object that encapsulates the flow context for the application purpose.
+ */
+ public static class FlowContext {
+ private final String flowName;
+ private final String flowVersion;
+ private final long flowRunId;
+
+ public FlowContext(String flowName, String flowVersion, long flowRunId) {
+ this.flowName = flowName;
+ this.flowVersion = flowVersion;
+ this.flowRunId = flowRunId;
+ }
+
+ public String getFlowName() {
+ return flowName;
+ }
+
+ public String getFlowVersion() {
+ return flowVersion;
+ }
+
+ public long getFlowRunId() {
+ return flowRunId;
+ }
+ }
+
private void createAndStartTimelineClient(Configuration conf) {
// create and start timeline client
this.timelineClient = TimelineClient.createTimelineClient(appId);
@@ -454,7 +488,11 @@ public class ApplicationImpl implements Application {
// Remove collectors info for finished apps.
// TODO check we remove related collectors info in failure cases
// (YARN-3038)
- app.context.getRegisteredCollectors().remove(app.getAppId());
+ Map<ApplicationId, String> registeredCollectors =
+ app.context.getRegisteredCollectors();
+ if (registeredCollectors != null) {
+ registeredCollectors.remove(app.getAppId());
+ }
// stop timelineClient when application get finished.
TimelineClient timelineClient = app.getTimelineClient();
if (timelineClient != null) {
@@ -521,16 +559,16 @@ public class ApplicationImpl implements Application {
@Override
public String getFlowName() {
- return flowName;
+ return flowContext == null ? null : flowContext.getFlowName();
}
@Override
public String getFlowVersion() {
- return flowVersion;
+ return flowContext == null ? null : flowContext.getFlowVersion();
}
@Override
public long getFlowRunId() {
- return flowRunId;
+ return flowContext == null ? 0L : flowContext.getFlowRunId();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 40c5996..739d363 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
@@ -564,9 +565,13 @@ public class ContainersMonitorImpl extends AbstractService implements
ContainerImpl container =
(ContainerImpl) context.getContainers().get(containerId);
- container.getNMTimelinePublisher().reportContainerResourceUsage(
- container, currentTime, pId, currentPmemUsage,
- cpuUsageTotalCoresPercentage);
+ NMTimelinePublisher nmMetricsPublisher =
+ container.getNMTimelinePublisher();
+ if (nmMetricsPublisher != null) {
+ nmMetricsPublisher.reportContainerResourceUsage(
+ container, currentTime, pId, currentPmemUsage,
+ cpuUsageTotalCoresPercentage);
+ }
} catch (Exception e) {
// Log the exception and proceed to the next container.
LOG.warn("Uncaught exception in ContainersMonitorImpl "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 2c5c300..69de433 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.TimelineClient;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -56,12 +55,16 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.Contai
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+/**
+ * Metrics publisher service that publishes data to the timeline service v.2. It
+ * is used only if the timeline service v.2 is enabled and the system publishing
+ * of events and metrics is enabled.
+ */
public class NMTimelinePublisher extends CompositeService {
private static final Log LOG = LogFactory.getLog(NMTimelinePublisher.class);
private Dispatcher dispatcher;
- private boolean publishSystemMetrics;
private Context context;
@@ -76,24 +79,16 @@ public class NMTimelinePublisher extends CompositeService {
@Override
protected void serviceInit(Configuration conf) throws Exception {
- publishSystemMetrics =
- YarnConfiguration.systemMetricsPublisherEnabled(conf);
-
- if (publishSystemMetrics) {
- dispatcher = new AsyncDispatcher();
- dispatcher.register(NMTimelineEventType.class,
- new ForwardingEventHandler());
- dispatcher
- .register(ContainerEventType.class, new ContainerEventHandler());
- dispatcher.register(ApplicationEventType.class,
- new ApplicationEventHandler());
- dispatcher.register(LocalizationEventType.class,
- new LocalizationEventDispatcher());
- addIfService(dispatcher);
- LOG.info("YARN system metrics publishing service is enabled");
- } else {
- LOG.info("YARN system metrics publishing service is not enabled");
- }
+ dispatcher = new AsyncDispatcher();
+ dispatcher.register(NMTimelineEventType.class,
+ new ForwardingEventHandler());
+ dispatcher
+ .register(ContainerEventType.class, new ContainerEventHandler());
+ dispatcher.register(ApplicationEventType.class,
+ new ApplicationEventHandler());
+ dispatcher.register(LocalizationEventType.class,
+ new LocalizationEventDispatcher());
+ addIfService(dispatcher);
super.serviceInit(conf);
}
@@ -121,8 +116,9 @@ public class NMTimelinePublisher extends CompositeService {
public void reportContainerResourceUsage(Container container,
long createdTime, String pId, Long pmemUsage,
Float cpuUsageTotalCoresPercentage) {
- if (publishSystemMetrics
- && (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE || cpuUsageTotalCoresPercentage != ResourceCalculatorProcessTree.UNAVAILABLE)) {
+ if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE ||
+ cpuUsageTotalCoresPercentage !=
+ ResourceCalculatorProcessTree.UNAVAILABLE) {
ContainerEntity entity =
createContainerEntity(container.getContainerId());
long currentTimeMillis = System.currentTimeMillis();
@@ -219,9 +215,6 @@ public class NMTimelinePublisher extends CompositeService {
}
public void publishApplicationEvent(ApplicationEvent event) {
- if (!publishSystemMetrics) {
- return;
- }
// publish only when the desired event is received
switch (event.getType()) {
case INIT_APPLICATION:
@@ -242,9 +235,6 @@ public class NMTimelinePublisher extends CompositeService {
}
public void publishContainerEvent(ContainerEvent event) {
- if (!publishSystemMetrics) {
- return;
- }
// publish only when the desired event is received
switch (event.getType()) {
case INIT_CONTAINER:
@@ -262,9 +252,6 @@ public class NMTimelinePublisher extends CompositeService {
}
public void publishLocalizationEvent(LocalizationEvent event) {
- if (!publishSystemMetrics) {
- return;
- }
// publish only when the desired event is received
switch (event.getType()) {
case CONTAINER_RESOURCES_LOCALIZED:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef71c1fc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
index 13956db..9b3cff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManagerRecovery.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
@@ -95,7 +94,6 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.Conta
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
-import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMMemoryStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
@@ -106,7 +104,6 @@ import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.junit.Before;
import org.junit.Test;
-import org.mockito.Mockito;
public class TestContainerManagerRecovery extends BaseContainerManagerTest {
@@ -642,9 +639,9 @@ public class TestContainerManagerRecovery extends BaseContainerManagerTest {
}
@Override
- public NMTimelinePublisher createNMTimelinePublisher(Context context) {
- NMTimelinePublisher timelinePublisher = mock(NMTimelinePublisher.class);
- return timelinePublisher;
+ public NMTimelinePublisher
+ createNMTimelinePublisher(Context context) {
+ return null;
}
};
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-3863. Support complex filters
in TimelineReader (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index d8f73d4..6696ac5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -19,13 +19,8 @@ package org.apache.hadoop.yarn.server.timelineservice.storage.reader;
import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
@@ -33,28 +28,22 @@ import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
import org.apache.hadoop.hbase.filter.FamilyFilter;
import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.QualifierFilter;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
@@ -71,7 +60,6 @@ import com.google.common.base.Preconditions;
*/
class GenericEntityReader extends TimelineEntityReader {
private static final EntityTable ENTITY_TABLE = new EntityTable();
- private static final Log LOG = LogFactory.getLog(GenericEntityReader.class);
/**
* Used to look up the flow context.
@@ -97,92 +85,322 @@ class GenericEntityReader extends TimelineEntityReader {
}
@Override
- protected FilterList constructFilterListBasedOnFields() {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
- TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
- // Fetch all the columns.
- if (dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (dataToRetrieve.getConfsToRetrieve() == null ||
- dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) &&
- (dataToRetrieve.getMetricsToRetrieve() == null ||
- dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
- return list;
+ protected FilterList constructFilterListBasedOnFilters() throws IOException {
+ // Filters here cannot be null for multiple entity reads as they are set in
+ // augmentParams if null.
+ FilterList listBasedOnFilters = new FilterList();
+ TimelineEntityFilters filters = getFilters();
+ // Create filter list based on created time range and add it to
+ // listBasedOnFilters.
+ long createdTimeBegin = filters.getCreatedTimeBegin();
+ long createdTimeEnd = filters.getCreatedTimeEnd();
+ if (createdTimeBegin != 0 || createdTimeEnd != Long.MAX_VALUE) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createSingleColValueFiltersByRange(
+ EntityColumn.CREATED_TIME, createdTimeBegin, createdTimeEnd));
}
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(EntityColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
+ // Create filter list based on metric filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList metricFilters = filters.getMetricFilters();
+ if (metricFilters != null && !metricFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.METRIC, metricFilters));
+ }
+ // Create filter list based on config filters and add it to
+ // listBasedOnFilters.
+ TimelineFilterList configFilters = filters.getConfigFilters();
+ if (configFilters != null && !configFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.CONFIG, configFilters));
+ }
+ // Create filter list based on info filters and add it to listBasedOnFilters
+ TimelineFilterList infoFilters = filters.getInfoFilters();
+ if (infoFilters != null && !infoFilters.getFilterList().isEmpty()) {
+ listBasedOnFilters.addFilter(
+ TimelineFilterUtils.createHBaseFilterList(
+ EntityColumnPrefix.INFO, infoFilters));
+ }
+ return listBasedOnFilters;
+ }
+
+ /**
+ * Check if we need to fetch only some of the event columns.
+ *
+ * @return true if we need to fetch some of the columns, false otherwise.
+ */
+ private static boolean fetchPartialEventCols(TimelineFilterList eventFilters,
+ EnumSet<Field> fieldsToRetrieve) {
+ return (eventFilters != null && !eventFilters.getFilterList().isEmpty() &&
+ !TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS));
+ }
+
+ /**
+ * Check if we need to fetch only some of the relates_to columns.
+ *
+ * @return true if we need to fetch some of the columns, false otherwise.
+ */
+ private static boolean fetchPartialRelatesToCols(TimelineFilterList relatesTo,
+ EnumSet<Field> fieldsToRetrieve) {
+ return (relatesTo != null && !relatesTo.getFilterList().isEmpty() &&
+ !TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO));
+ }
+
+ /**
+ * Check if we need to fetch only some of the is_related_to columns.
+ *
+ * @return true if we need to fetch some of the columns, false otherwise.
+ */
+ private static boolean fetchPartialIsRelatedToCols(
+ TimelineFilterList isRelatedTo, EnumSet<Field> fieldsToRetrieve) {
+ return (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty() &&
+ !TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO));
+ }
+
+ /**
+ * Check if we need to fetch only some of the columns based on event filters,
+ * relatesto and isrelatedto from info family.
+ *
+ * @return true, if we need to fetch only some of the columns, false if we
+ * need to fetch all the columns under info column family.
+ */
+ protected boolean fetchPartialColsFromInfoFamily() {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
TimelineEntityFilters filters = getFilters();
+ return fetchPartialEventCols(filters.getEventFilters(), fieldsToRetrieve) ||
+ fetchPartialRelatesToCols(filters.getRelatesTo(), fieldsToRetrieve) ||
+ fetchPartialIsRelatedToCols(filters.getIsRelatedTo(), fieldsToRetrieve);
+ }
+
+ /**
+ * Check if we need to create filter list based on fields. We need to create
+ * a filter list iff all fields need not be retrieved or we have some specific
+ * fields or metrics to retrieve. We also need to create a filter list if we
+ * have relationships(relatesTo/isRelatedTo) and event filters specified for
+ * the query.
+ *
+ * @return true if we need to create the filter list, false otherwise.
+ */
+ protected boolean needCreateFilterListBasedOnFields() {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Check if all fields are to be retrieved or not. If all fields have to
+ // be retrieved, also check if we have some metrics or configs to
+ // retrieve specified for the query because then a filter list will have
+ // to be created.
+ boolean flag = !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) ||
+ (dataToRetrieve.getConfsToRetrieve() != null &&
+ !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) ||
+ (dataToRetrieve.getMetricsToRetrieve() != null &&
+ !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty());
+ // Filters need to be checked only if we are reading multiple entities. If
+ // condition above is false, we check if there are relationships(relatesTo/
+ // isRelatedTo) and event filters specified for the query.
+ if (!flag && !isSingleEntityRead()) {
+ TimelineEntityFilters filters = getFilters();
+ flag = (filters.getEventFilters() != null &&
+ !filters.getEventFilters().getFilterList().isEmpty()) ||
+ (filters.getIsRelatedTo() != null &&
+ !filters.getIsRelatedTo().getFilterList().isEmpty()) ||
+ (filters.getRelatesTo() != null &&
+ !filters.getRelatesTo().getFilterList().isEmpty());
+ }
+ return flag;
+ }
+
+ /**
+ * Add {@link QualifierFilter} filters to filter list for each column of
+ * entity table.
+ *
+ * @param list filter list to which qualifier filters have to be added.
+ */
+ protected void updateFixedColumns(FilterList list) {
+ for (EntityColumn column : EntityColumn.values()) {
+ list.addFilter(new QualifierFilter(CompareOp.EQUAL,
+ new BinaryComparator(column.getColumnQualifierBytes())));
+ }
+ }
+
+ /**
+ * Creates a filter list which indicates that only some of the column
+ * qualifiers in the info column family will be returned in result.
+ *
+ * @param isApplication If true, it means operations are to be performed for
+ * application table, otherwise for entity table.
+ * @return filter list.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterListForColsOfInfoFamily()
+ throws IOException {
+ FilterList infoFamilyColsFilter = new FilterList(Operator.MUST_PASS_ONE);
+ // Add filters for each column in entity table.
+ updateFixedColumns(infoFamilyColsFilter);
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
+ // If INFO field has to be retrieved, add a filter for fetching columns
+ // with INFO column prefix.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.INFO)) {
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, EntityColumnPrefix.INFO));
+ }
+ TimelineFilterList relatesTo = getFilters().getRelatesTo();
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO)) {
+ // If RELATES_TO field has to be retrieved, add a filter for fetching
+ // columns with RELATES_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, EntityColumnPrefix.RELATES_TO));
+ } else if (relatesTo != null && !relatesTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain RELATES_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // relatesTo filters are specified. relatesTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> relatesToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(relatesTo);
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createFiltersFromColumnQualifiers(
+ EntityColumnPrefix.RELATES_TO, relatesToCols));
+ }
+ TimelineFilterList isRelatedTo = getFilters().getIsRelatedTo();
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
+ // If IS_RELATED_TO field has to be retrieved, add a filter for fetching
+ // columns with IS_RELATED_TO column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, EntityColumnPrefix.IS_RELATED_TO));
+ } else if (isRelatedTo != null && !isRelatedTo.getFilterList().isEmpty()) {
+ // Even if fields to retrieve does not contain IS_RELATED_TO, we still
+ // need to have a filter to fetch some of the column qualifiers if
+ // isRelatedTo filters are specified. isRelatedTo filters will then be
+ // matched after fetching rows from HBase.
+ Set<String> isRelatedToCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(isRelatedTo);
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createFiltersFromColumnQualifiers(
+ EntityColumnPrefix.IS_RELATED_TO, isRelatedToCols));
+ }
+ TimelineFilterList eventFilters = getFilters().getEventFilters();
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS)) {
+ // If EVENTS field has to be retrieved, add a filter for fetching columns
+ // with EVENT column prefix.
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.EQUAL, EntityColumnPrefix.EVENT));
+ } else if (eventFilters != null && !eventFilters.getFilterList().isEmpty()){
+ // Even if fields to retrieve does not contain EVENTS, we still need to
+ // have a filter to fetch some of the column qualifiers on the basis of
+ // event filters specified. Event filters will then be matched after
+ // fetching rows from HBase.
+ Set<String> eventCols =
+ TimelineFilterUtils.fetchColumnsFromFilterList(eventFilters);
+ infoFamilyColsFilter.addFilter(
+ TimelineFilterUtils.createFiltersFromColumnQualifiers(
+ EntityColumnPrefix.EVENT, eventCols));
+ }
+ return infoFamilyColsFilter;
+ }
+
+ /**
+ * Exclude column prefixes via filters which are not required(based on fields
+ * to retrieve) from info column family. These filters are added to filter
+ * list which contains a filter for getting info column family.
+ *
+ * @param infoColFamilyList filter list for info column family.
+ */
+ private void excludeFieldsFromInfoColFamily(FilterList infoColFamilyList) {
+ EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
// Events not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.EVENTS) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getEventFilters() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.EVENT.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, EntityColumnPrefix.EVENT));
}
// info not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.INFO) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getInfoFilters() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.INFO)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.INFO.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, EntityColumnPrefix.INFO));
}
// is related to not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.IS_RELATED_TO) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getIsRelatedTo() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, EntityColumnPrefix.IS_RELATED_TO));
}
// relates to not required.
- if (!dataToRetrieve.getFieldsToRetrieve().contains(Field.RELATES_TO) &&
- !dataToRetrieve.getFieldsToRetrieve().contains(Field.ALL) &&
- (isSingleEntityRead() || filters.getRelatesTo() == null)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO)) {
infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
+ TimelineFilterUtils.createHBaseQualifierFilter(
+ CompareOp.NOT_EQUAL, EntityColumnPrefix.RELATES_TO));
}
- list.addFilter(infoColFamilyList);
- if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS) ||
- (!isSingleEntityRead() && filters.getConfigFilters() != null)) ||
- (dataToRetrieve.getConfsToRetrieve() != null &&
- !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty())) {
- FilterList filterCfg =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(EntityColumnFamily.CONFIGS.getBytes())));
- if (dataToRetrieve.getConfsToRetrieve() != null &&
- !dataToRetrieve.getConfsToRetrieve().getFilterList().isEmpty()) {
- filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.CONFIG, dataToRetrieve.getConfsToRetrieve()));
- }
- list.addFilter(filterCfg);
+ }
+
+ /**
+ * Updates filter list based on fields for confs and metrics to retrieve.
+ *
+ * @param listBasedOnFields filter list based on fields.
+ * @throws IOException if any problem occurs while updating filter list.
+ */
+ private void updateFilterForConfsAndMetricsToRetrieve(
+ FilterList listBasedOnFields) throws IOException {
+ TimelineDataToRetrieve dataToRetrieve = getDataToRetrieve();
+ // Please note that if confsToRetrieve is specified, we would have added
+ // CONFS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.CONFIGS)) {
+ // Create a filter list for configs.
+ listBasedOnFields.addFilter(TimelineFilterUtils.
+ createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getConfsToRetrieve(),
+ EntityColumnFamily.CONFIGS, EntityColumnPrefix.CONFIG));
}
- if ((dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS) ||
- (!isSingleEntityRead() && filters.getMetricFilters() != null)) ||
- (dataToRetrieve.getMetricsToRetrieve() != null &&
- !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty())) {
- FilterList filterMetrics =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(EntityColumnFamily.METRICS.getBytes())));
- if (dataToRetrieve.getMetricsToRetrieve() != null &&
- !dataToRetrieve.getMetricsToRetrieve().getFilterList().isEmpty()) {
- filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.METRIC, dataToRetrieve.getMetricsToRetrieve()));
- }
- list.addFilter(filterMetrics);
+
+ // Please note that if metricsToRetrieve is specified, we would have added
+ // METRICS to fields to retrieve in augmentParams() even if not specified.
+ if (dataToRetrieve.getFieldsToRetrieve().contains(Field.METRICS)) {
+ // Create a filter list for metrics.
+ listBasedOnFields.addFilter(TimelineFilterUtils.
+ createFilterForConfsOrMetricsToRetrieve(
+ dataToRetrieve.getMetricsToRetrieve(),
+ EntityColumnFamily.METRICS, EntityColumnPrefix.METRIC));
+ }
+ }
+
+ @Override
+ protected FilterList constructFilterListBasedOnFields() throws IOException {
+ if (!needCreateFilterListBasedOnFields()) {
+ // Fetch all the columns. No need of a filter.
+ return null;
+ }
+ FilterList listBasedOnFields = new FilterList(Operator.MUST_PASS_ONE);
+ FilterList infoColFamilyList = new FilterList();
+ // By default fetch everything in INFO column family.
+ FamilyFilter infoColumnFamily =
+ new FamilyFilter(CompareOp.EQUAL,
+ new BinaryComparator(EntityColumnFamily.INFO.getBytes()));
+ infoColFamilyList.addFilter(infoColumnFamily);
+ if (!isSingleEntityRead() && fetchPartialColsFromInfoFamily()) {
+ // We can fetch only some of the columns from info family.
+ infoColFamilyList.addFilter(createFilterListForColsOfInfoFamily());
+ } else {
+ // Exclude column prefixes in info column family which are not required
+ // based on fields to retrieve.
+ excludeFieldsFromInfoColFamily(infoColFamilyList);
}
- return list;
+ listBasedOnFields.addFilter(infoColFamilyList);
+ updateFilterForConfsAndMetricsToRetrieve(listBasedOnFields);
+ return listBasedOnFields;
}
+ /**
+ * Looks up flow context from AppToFlow table.
+ *
+ * @param clusterId Cluster Id.
+ * @param appId App Id.
+ * @param hbaseConf HBase configuration.
+ * @param conn HBase Connection.
+ * @return flow context information.
+ * @throws IOException if any problem occurs while fetching flow information.
+ */
protected FlowContext lookupFlowContext(String clusterId, String appId,
Configuration hbaseConf, Connection conn) throws IOException {
byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
@@ -200,6 +418,9 @@ class GenericEntityReader extends TimelineEntityReader {
}
}
+ /**
+ * Encapsulates flow context information.
+ */
protected static class FlowContext {
private final String userId;
private final String flowName;
@@ -222,6 +443,9 @@ class GenericEntityReader extends TimelineEntityReader {
@Override
protected void validateParams() {
+ Preconditions.checkNotNull(getContext(), "context shouldn't be null");
+ Preconditions.checkNotNull(
+ getDataToRetrieve(), "data to retrieve shouldn't be null");
Preconditions.checkNotNull(getContext().getClusterId(),
"clusterId shouldn't be null");
Preconditions.checkNotNull(getContext().getAppId(),
@@ -241,13 +465,19 @@ class GenericEntityReader extends TimelineEntityReader {
// In reality all three should be null or neither should be null
if (context.getFlowName() == null || context.getFlowRunId() == null ||
context.getUserId() == null) {
+ // Get flow context information from AppToFlow table.
FlowContext flowContext = lookupFlowContext(
context.getClusterId(), context.getAppId(), hbaseConf, conn);
context.setFlowName(flowContext.flowName);
context.setFlowRunId(flowContext.flowRunId);
context.setUserId(flowContext.userId);
}
+ // Add configs/metrics to fields to retrieve if confsToRetrieve and/or
+ // metricsToRetrieve are specified.
getDataToRetrieve().addFieldsBasedOnConfsAndMetricsToRetrieve();
+ if (!isSingleEntityRead()) {
+ createFiltersIfNull();
+ }
}
@Override
@@ -298,215 +528,84 @@ class GenericEntityReader extends TimelineEntityReader {
// fetch created time
Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result);
entity.setCreatedTime(createdTime.longValue());
- if (!isSingleEntityRead() &&
- (entity.getCreatedTime() < filters.getCreatedTimeBegin() ||
- entity.getCreatedTime() > filters.getCreatedTimeEnd())) {
- return null;
- }
+
EnumSet<Field> fieldsToRetrieve = getDataToRetrieve().getFieldsToRetrieve();
- // fetch is related to entities
+ // fetch is related to entities and match isRelatedTo filter. If isRelatedTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // isRelatedTo are not set in HBase scan.
boolean checkIsRelatedTo =
- filters != null && filters.getIsRelatedTo() != null &&
- filters.getIsRelatedTo().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
- if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), filters.getIsRelatedTo())) {
+ !isSingleEntityRead() && filters.getIsRelatedTo() != null &&
+ filters.getIsRelatedTo().getFilterList().size() > 0;
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.IS_RELATED_TO) ||
+ checkIsRelatedTo) {
+ TimelineStorageUtils.readRelationship(
+ entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
+ if (checkIsRelatedTo && !TimelineStorageUtils.matchIsRelatedTo(entity,
+ filters.getIsRelatedTo())) {
return null;
}
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve,
+ Field.IS_RELATED_TO)) {
entity.getIsRelatedToEntities().clear();
}
}
- // fetch relates to entities
+ // fetch relates to entities and match relatesTo filter. If relatesTo
+ // filters do not match, entity would be dropped. We have to match filters
+ // locally as relevant HBase filters to filter out rows on the basis of
+ // relatesTo are not set in HBase scan.
boolean checkRelatesTo =
- filters != null && filters.getRelatesTo() != null &&
- filters.getRelatesTo().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
- if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), filters.getRelatesTo())) {
+ !isSingleEntityRead() && filters.getRelatesTo() != null &&
+ filters.getRelatesTo().getFilterList().size() > 0;
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO) ||
+ checkRelatesTo) {
+ TimelineStorageUtils.readRelationship(
+ entity, result, EntityColumnPrefix.RELATES_TO, false);
+ if (checkRelatesTo && !TimelineStorageUtils.matchRelatesTo(entity,
+ filters.getRelatesTo())) {
return null;
}
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.RELATES_TO)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.RELATES_TO)) {
entity.getRelatesToEntities().clear();
}
}
- // fetch info
- boolean checkInfo = filters != null && filters.getInfoFilters() != null &&
- filters.getInfoFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
- if (checkInfo &&
- !TimelineStorageUtils.matchFilters(
- entity.getInfo(), filters.getInfoFilters())) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.INFO)) {
- entity.getInfo().clear();
- }
+ // fetch info if fieldsToRetrieve contains INFO or ALL.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.INFO)) {
+ TimelineStorageUtils.readKeyValuePairs(
+ entity, result, EntityColumnPrefix.INFO, false);
}
- // fetch configs
- boolean checkConfigs =
- filters != null && filters.getConfigFilters() != null &&
- filters.getConfigFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
- if (checkConfigs && !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), filters.getConfigFilters())) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.CONFIGS)) {
- entity.getConfigs().clear();
- }
+ // fetch configs if fieldsToRetrieve contains CONFIGS or ALL.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.CONFIGS)) {
+ TimelineStorageUtils.readKeyValuePairs(
+ entity, result, EntityColumnPrefix.CONFIG, true);
}
- // fetch events
+ // fetch events and match event filters if they exist. If event filters do
+ // not match, entity would be dropped. We have to match filters locally
+ // as relevant HBase filters to filter out rows on the basis of events
+ // are not set in HBase scan.
boolean checkEvents =
- filters != null && filters.getEventFilters() != null &&
- filters.getEventFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result, false);
- if (checkEvents && !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), filters.getEventFilters())) {
+ !isSingleEntityRead() && filters.getEventFilters() != null &&
+ filters.getEventFilters().getFilterList().size() > 0;
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS) ||
+ checkEvents) {
+ TimelineStorageUtils.readEvents(entity, result, EntityColumnPrefix.EVENT);
+ if (checkEvents && !TimelineStorageUtils.matchEventFilters(entity,
+ filters.getEventFilters())) {
return null;
}
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.EVENTS)) {
+ if (!TimelineStorageUtils.hasField(fieldsToRetrieve, Field.EVENTS)) {
entity.getEvents().clear();
}
}
- // fetch metrics
- boolean checkMetrics =
- filters != null && filters.getMetricFilters() != null &&
- filters.getMetricFilters().size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
+ // fetch metrics if fieldsToRetrieve contains METRICS or ALL.
+ if (TimelineStorageUtils.hasField(fieldsToRetrieve, Field.METRICS)) {
readMetrics(entity, result, EntityColumnPrefix.METRIC);
- if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), filters.getMetricFilters())) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.METRICS)) {
- entity.getMetrics().clear();
- }
}
return entity;
}
-
- /**
- * Helper method for reading relationship.
- *
- * @param <T> Describes the type of column prefix.
- * @param entity entity to fill.
- * @param result result from HBase.
- * @param prefix column prefix.
- * @param isRelatedTo if true, means relationship is to be added to
- * isRelatedTo, otherwise its added to relatesTo.
- * @throws IOException if any problem is encountered while reading result.
- */
- protected <T> void readRelationship(
- TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
- boolean isRelatedTo) throws IOException {
- // isRelatedTo and relatesTo are of type Map<String, Set<String>>
- Map<String, Object> columns = prefix.readResults(result);
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- for (String id : Separator.VALUES.splitEncoded(
- column.getValue().toString())) {
- if (isRelatedTo) {
- entity.addIsRelatedToEntity(column.getKey(), id);
- } else {
- entity.addRelatesToEntity(column.getKey(), id);
- }
- }
- }
- }
-
- /**
- * Helper method for reading key-value pairs for either info or config.
- *
- * @param <T> Describes the type of column prefix.
- * @param entity entity to fill.
- * @param result result from HBase.
- * @param prefix column prefix.
- * @param isConfig if true, means we are reading configs, otherwise info.
- * @throws IOException if any problem is encountered while reading result.
- */
- protected <T> void readKeyValuePairs(
- TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
- boolean isConfig) throws IOException {
- // info and configuration are of type Map<String, Object or String>
- Map<String, Object> columns = prefix.readResults(result);
- if (isConfig) {
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- entity.addConfig(column.getKey(), column.getValue().toString());
- }
- } else {
- entity.addInfo(columns);
- }
- }
-
- /**
- * Read events from the entity table or the application table. The column name
- * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
- * if there is no info associated with the event.
- *
- * @param entity entity to fill.
- * @param result HBase Result.
- * @param isApplication if true, event read is for application table,
- * otherwise its being read for entity table.
- * @throws IOException if any problem is encountered while reading result.
- *
- * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
- * schema description.
- */
- protected void readEvents(TimelineEntity entity, Result result,
- boolean isApplication) throws IOException {
- Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<?, Object> eventsResult = isApplication ?
- ApplicationColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result) :
- EntityColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
- byte[][] karr = (byte[][])eventResult.getKey();
- // the column name is of the form "eventId=timestamp=infoKey"
- if (karr.length == 3) {
- String id = Bytes.toString(karr[0]);
- long ts = TimelineStorageUtils.invertLong(Bytes.toLong(karr[1]));
- String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
- TimelineEvent event = eventsMap.get(key);
- if (event == null) {
- event = new TimelineEvent();
- event.setId(id);
- event.setTimestamp(ts);
- eventsMap.put(key, event);
- }
- // handle empty info
- String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
- if (infoKey != null) {
- event.addInfo(infoKey, eventResult.getValue());
- }
- } else {
- LOG.warn("incorrectly formatted column name: it will be discarded");
- continue;
- }
- }
- Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
- entity.addEvents(eventsSet);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
index 281e901..4299de9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/TimelineEntityReader.java
@@ -107,11 +107,60 @@ public abstract class TimelineEntityReader {
/**
* Creates a {@link FilterList} based on fields, confs and metrics to
* retrieve. This filter list will be set in Scan/Get objects to trim down
- * results fetched from HBase back-end storage.
+ * results fetched from HBase back-end storage. This is called only for
+ * multiple entity reads.
*
* @return a {@link FilterList} object.
+ * @throws IOException if any problem occurs while creating filter list.
*/
- protected abstract FilterList constructFilterListBasedOnFields();
+ protected abstract FilterList constructFilterListBasedOnFields()
+ throws IOException;
+
+ /**
+ * Creates a {@link FilterList} based on info, config and metric filters. This
+ * filter list will be set in HBase Get to trim down results fetched from
+ * HBase back-end storage.
+ *
+ * @return a {@link FilterList} object.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ protected abstract FilterList constructFilterListBasedOnFilters()
+ throws IOException;
+
+ /**
+ * Combines filter lists created based on fields and based on filters.
+ *
+ * @return a {@link FilterList} object if it can be constructed. Returns null,
+ * if filter list cannot be created either on the basis of filters or on the
+ * basis of fields.
+ * @throws IOException if any problem occurs while creating filter list.
+ */
+ private FilterList createFilterList() throws IOException {
+ FilterList listBasedOnFilters = constructFilterListBasedOnFilters();
+ boolean hasListBasedOnFilters = listBasedOnFilters != null &&
+ !listBasedOnFilters.getFilters().isEmpty();
+ FilterList listBasedOnFields = constructFilterListBasedOnFields();
+ boolean hasListBasedOnFields = listBasedOnFields != null &&
+ !listBasedOnFields.getFilters().isEmpty();
+ // If filter lists based on both filters and fields can be created,
+ // combine them in a new filter list and return it.
+ // If either one of them has been created, return that filter list.
+ // Return null, if none of the filter lists can be created. This indicates
+ // that no filter list needs to be added to HBase Scan as filters are not
+ // specified for the query or only the default view of entity needs to be
+ // returned.
+ if (hasListBasedOnFilters && hasListBasedOnFields) {
+ FilterList list = new FilterList();
+ list.addFilter(listBasedOnFilters);
+ list.addFilter(listBasedOnFields);
+ return list;
+ } else if (hasListBasedOnFilters) {
+ return listBasedOnFilters;
+ } else if (hasListBasedOnFields) {
+ return listBasedOnFields;
+ }
+ return null;
+ }
protected TimelineReaderContext getContext() {
return context;
@@ -126,6 +175,16 @@ public abstract class TimelineEntityReader {
}
/**
+ * Create a {@link TimelineEntityFilters} object with default values for
+ * filters.
+ */
+ protected void createFiltersIfNull() {
+ if (filters == null) {
+ filters = new TimelineEntityFilters();
+ }
+ }
+
+ /**
* Reads and deserializes a single timeline entity from the HBase storage.
*
* @param hbaseConf HBase Configuration.
@@ -140,6 +199,9 @@ public abstract class TimelineEntityReader {
augmentParams(hbaseConf, conn);
FilterList filterList = constructFilterListBasedOnFields();
+ if (LOG.isDebugEnabled() && filterList != null) {
+ LOG.debug("FilterList created for get is - " + filterList);
+ }
Result result = getResult(hbaseConf, conn, filterList);
if (result == null || result.isEmpty()) {
// Could not find a matching row.
@@ -166,7 +228,10 @@ public abstract class TimelineEntityReader {
augmentParams(hbaseConf, conn);
NavigableSet<TimelineEntity> entities = new TreeSet<>();
- FilterList filterList = constructFilterListBasedOnFields();
+ FilterList filterList = createFilterList();
+ if (LOG.isDebugEnabled() && filterList != null) {
+ LOG.debug("FilterList created for scan is - " + filterList);
+ }
ResultScanner results = getResults(hbaseConf, conn, filterList);
try {
for (Result result : results) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index b6e23a9..2bd2830 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -221,7 +221,7 @@ public class TestTimelineReaderWebServices {
assertTrue("UID should be present",
entity.getInfo().containsKey(TimelineReaderManager.UID_KEY));
// Includes UID.
- assertEquals(2, entity.getInfo().size());
+ assertEquals(3, entity.getInfo().size());
// No events will be returned as events are not part of fields.
assertEquals(0, entity.getEvents().size());
} finally {
@@ -247,7 +247,7 @@ public class TestTimelineReaderWebServices {
assertTrue("UID should be present",
entity.getInfo().containsKey(TimelineReaderManager.UID_KEY));
// Includes UID.
- assertEquals(2, entity.getInfo().size());
+ assertEquals(3, entity.getInfo().size());
assertEquals(2, entity.getEvents().size());
} finally {
client.destroy();
@@ -443,10 +443,8 @@ public class TestTimelineReaderWebServices {
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
assertNotNull(entities);
- assertEquals(2, entities.size());
- assertTrue("Entities with id_1 and id_3 should have been present" +
- " in response.",
- entities.contains(newEntity("app", "id_1")) &&
+ assertEquals(1, entities.size());
+ assertTrue("Entity with id_3 should have been present in response.",
entities.contains(newEntity("app", "id_3")));
} finally {
client.destroy();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b2df86c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index a8a2ff8..23d64e0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -23,6 +23,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.Arrays;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
@@ -40,6 +41,13 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineDataToRetrieve;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineEntityFilters;
import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContext;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.AfterClass;
@@ -112,6 +120,7 @@ public class TestFileSystemTimelineReaderImpl {
entity11.setCreatedTime(1425016502000L);
Map<String, Object> info1 = new HashMap<String, Object>();
info1.put("info1", "val1");
+ info1.put("info2", "val5");
entity11.addInfo(info1);
TimelineEvent event = new TimelineEvent();
event.setId("event_1");
@@ -121,7 +130,7 @@ public class TestFileSystemTimelineReaderImpl {
TimelineMetric metric1 = new TimelineMetric();
metric1.setId("metric1");
metric1.setType(TimelineMetric.Type.SINGLE_VALUE);
- metric1.addValue(1425016502006L, 113.2F);
+ metric1.addValue(1425016502006L, 113);
metrics.add(metric1);
TimelineMetric metric2 = new TimelineMetric();
metric2.setId("metric2");
@@ -130,7 +139,7 @@ public class TestFileSystemTimelineReaderImpl {
metrics.add(metric2);
entity11.setMetrics(metrics);
Map<String,String> configs = new HashMap<String, String>();
- configs.put("config_1", "123");
+ configs.put("config_1", "127");
entity11.setConfigs(configs);
entity11.addRelatesToEntity("flow", "flow1");
entity11.addIsRelatedToEntity("type1", "tid1_1");
@@ -171,7 +180,7 @@ public class TestFileSystemTimelineReaderImpl {
info1.put("info2", 4);
entity2.addInfo(info2);
Map<String,String> configs2 = new HashMap<String, String>();
- configs2.put("config_1", "123");
+ configs2.put("config_1", "129");
configs2.put("config_3", "def");
entity2.setConfigs(configs2);
TimelineEvent event2 = new TimelineEvent();
@@ -182,7 +191,7 @@ public class TestFileSystemTimelineReaderImpl {
TimelineMetric metric21 = new TimelineMetric();
metric21.setId("metric1");
metric21.setType(TimelineMetric.Type.SINGLE_VALUE);
- metric21.addValue(1425016501006L, 123.2F);
+ metric21.addValue(1425016501006L, 300);
metrics2.add(metric21);
TimelineMetric metric22 = new TimelineMetric();
metric22.setId("metric2");
@@ -205,6 +214,7 @@ public class TestFileSystemTimelineReaderImpl {
entity3.setCreatedTime(1425016501050L);
Map<String, Object> info3 = new HashMap<String, Object>();
info3.put("info2", 3.5);
+ info3.put("info4", 20);
entity3.addInfo(info3);
Map<String,String> configs3 = new HashMap<String, String>();
configs3.put("config_1", "123");
@@ -222,7 +232,7 @@ public class TestFileSystemTimelineReaderImpl {
TimelineMetric metric31 = new TimelineMetric();
metric31.setId("metric1");
metric31.setType(TimelineMetric.Type.SINGLE_VALUE);
- metric31.addValue(1425016501006L, 124.8F);
+ metric31.addValue(1425016501006L, 124);
metrics3.add(metric31);
TimelineMetric metric32 = new TimelineMetric();
metric32.setId("metric2");
@@ -317,7 +327,7 @@ public class TestFileSystemTimelineReaderImpl {
Assert.assertEquals(1425016502000L, result.getCreatedTime());
Assert.assertEquals(3, result.getConfigs().size());
Assert.assertEquals(3, result.getMetrics().size());
- Assert.assertEquals(1, result.getInfo().size());
+ Assert.assertEquals(2, result.getInfo().size());
// No events will be returned
Assert.assertEquals(0, result.getEvents().size());
}
@@ -344,8 +354,8 @@ public class TestFileSystemTimelineReaderImpl {
Set<TimelineEntity> result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null), new TimelineEntityFilters(),
- new TimelineDataToRetrieve());
- // All 3 entities will be returned
+ new TimelineDataToRetrieve(null, null, EnumSet.of(Field.ALL)));
+ // All 4 entities will be returned
Assert.assertEquals(4, result.size());
}
@@ -425,12 +435,13 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetFilteredEntities() throws Exception {
// Get entities based on info filters.
- Map<String, Object> infoFilters = new HashMap<String, Object>();
- infoFilters.put("info2", 3.5);
+ TimelineFilterList infoFilterList = new TimelineFilterList();
+ infoFilterList.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
Set<TimelineEntity> result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null),
- new TimelineEntityFilters(null, null, null, null, null, infoFilters,
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList,
null, null, null),
new TimelineDataToRetrieve());
Assert.assertEquals(1, result.size());
@@ -442,26 +453,30 @@ public class TestFileSystemTimelineReaderImpl {
}
// Get entities based on config filters.
- Map<String, String> configFilters = new HashMap<String, String>();
- configFilters.put("config_1", "123");
- configFilters.put("config_3", "abc");
+ TimelineFilterList confFilterList = new TimelineFilterList();
+ confFilterList.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_1", "123"));
+ confFilterList.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "abc"));
result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null),
new TimelineEntityFilters(null, null, null, null, null, null,
- configFilters, null, null),
+ confFilterList, null, null),
new TimelineDataToRetrieve());
- Assert.assertEquals(2, result.size());
+ Assert.assertEquals(1, result.size());
for (TimelineEntity entity : result) {
- if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
+ if (!entity.getId().equals("id_3")) {
Assert.fail("Incorrect filtering based on config filters");
}
}
// Get entities based on event filters.
- Set<String> eventFilters = new HashSet<String>();
- eventFilters.add("event_2");
- eventFilters.add("event_4");
+ TimelineFilterList eventFilters = new TimelineFilterList();
+ eventFilters.addFilter(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL,"event_2"));
+ eventFilters.addFilter(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL,"event_4"));
result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null),
@@ -476,13 +491,14 @@ public class TestFileSystemTimelineReaderImpl {
}
// Get entities based on metric filters.
- Set<String> metricFilters = new HashSet<String>();
- metricFilters.add("metric3");
+ TimelineFilterList metricFilterList = new TimelineFilterList();
+ metricFilterList.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, "metric3", 0L));
result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null),
new TimelineEntityFilters(null, null, null, null, null, null, null,
- metricFilters, null),
+ metricFilterList, null),
new TimelineDataToRetrieve());
Assert.assertEquals(2, result.size());
// Two entities with IDs' id_1 and id_2 should be returned.
@@ -491,15 +507,266 @@ public class TestFileSystemTimelineReaderImpl {
Assert.fail("Incorrect filtering based on metric filters");
}
}
- }
+
+ // Get entities based on complex config filters.
+ TimelineFilterList list1 = new TimelineFilterList();
+ list1.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_1", "129"));
+ list1.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "def"));
+ TimelineFilterList list2 = new TimelineFilterList();
+ list2.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_2", "23"));
+ list2.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_3", "abc"));
+ TimelineFilterList confFilterList1 =
+ new TimelineFilterList(Operator.OR, list1, list2);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList1, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on config filters");
+ }
+ }
+
+ TimelineFilterList list3 = new TimelineFilterList();
+ list3.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "config_1", "123"));
+ list3.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "config_3", "abc"));
+ TimelineFilterList list4 = new TimelineFilterList();
+ list4.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "config_2", "23"));
+ TimelineFilterList confFilterList2 =
+ new TimelineFilterList(Operator.OR, list3, list4);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList2, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on config filters");
+ }
+ }
+
+ TimelineFilterList confFilterList3 = new TimelineFilterList();
+ confFilterList3.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "config_1", "127"));
+ confFilterList3.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.NOT_EQUAL, "config_3", "abc"));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList3, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(1, result.size());
+ for(TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on config filters");
+ }
+ }
+
+ TimelineFilterList confFilterList4 = new TimelineFilterList();
+ confFilterList4.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_dummy", "dummy"));
+ confFilterList4.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_3", "def"));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList4, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(0, result.size());
+
+ TimelineFilterList confFilterList5 = new TimelineFilterList(Operator.OR);
+ confFilterList5.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_dummy", "dummy"));
+ confFilterList5.addFilter(new TimelineKeyValueFilter(
+ TimelineCompareOp.EQUAL, "config_3", "def"));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null,
+ confFilterList5, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on config filters");
+ }
+ }
+
+ // Get entities based on complex metric filters.
+ TimelineFilterList list6 = new TimelineFilterList();
+ list6.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_THAN, "metric1", 200));
+ list6.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.EQUAL, "metric3", 23));
+ TimelineFilterList list7 = new TimelineFilterList();
+ list7.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_OR_EQUAL, "metric2", 74));
+ TimelineFilterList metricFilterList1 =
+ new TimelineFilterList(Operator.OR, list6, list7);
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList1, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(2, result.size());
+ // Two entities with IDs' id_2 and id_3 should be returned.
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_2") && !entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on metric filters");
+ }
+ }
+
+ TimelineFilterList metricFilterList2 = new TimelineFilterList();
+ metricFilterList2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, "metric2", 70));
+ metricFilterList2.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList2, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1")) {
+ Assert.fail("Incorrect filtering based on metric filters");
+ }
+ }
+
+ TimelineFilterList metricFilterList3 = new TimelineFilterList();
+ metricFilterList3.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, "dummy_metric", 30));
+ metricFilterList3.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList3, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(0, result.size());
+
+ TimelineFilterList metricFilterList4 = new TimelineFilterList(Operator.OR);
+ metricFilterList4.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_THAN, "dummy_metric", 30));
+ metricFilterList4.addFilter(new TimelineCompareFilter(
+ TimelineCompareOp.LESS_OR_EQUAL, "metric3", 23));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList4, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on metric filters");
+ }
+ }
+
+ TimelineFilterList metricFilterList5 =
+ new TimelineFilterList(new TimelineCompareFilter(
+ TimelineCompareOp.NOT_EQUAL, "metric2", 74));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, null, null,
+ metricFilterList5, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_2")) {
+ Assert.fail("Incorrect filtering based on metric filters");
+ }
+ }
+
+ TimelineFilterList infoFilterList1 = new TimelineFilterList();
+ infoFilterList1.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
+ infoFilterList1.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL, "info4", 20));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList1,
+ null, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(0, result.size());
+
+ TimelineFilterList infoFilterList2 = new TimelineFilterList(Operator.OR);
+ infoFilterList2.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", 3.5));
+ infoFilterList2.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info1", "val1"));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList2,
+ null, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(2, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
+ Assert.fail("Incorrect filtering based on info filters");
+ }
+ }
+
+ TimelineFilterList infoFilterList3 = new TimelineFilterList();
+ infoFilterList3.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "dummy_info", 1));
+ infoFilterList3.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", "val5"));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList3,
+ null, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(0, result.size());
+
+ TimelineFilterList infoFilterList4 = new TimelineFilterList(Operator.OR);
+ infoFilterList4.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "dummy_info", 1));
+ infoFilterList4.addFilter(
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL, "info2", "val5"));
+ result = reader.getEntities(
+ new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
+ "app", null),
+ new TimelineEntityFilters(null, null, null, null, null, infoFilterList4,
+ null, null, null),
+ new TimelineDataToRetrieve());
+ Assert.assertEquals(1, result.size());
+ for (TimelineEntity entity : result) {
+ if (!entity.getId().equals("id_1")) {
+ Assert.fail("Incorrect filtering based on info filters");
+ }
+ }
+ }
@Test
public void testGetEntitiesByRelations() throws Exception {
// Get entities based on relatesTo.
- Map<String, Set<String>> relatesTo = new HashMap<String, Set<String>>();
- Set<String> relatesToIds = new HashSet<String>();
- relatesToIds.add("flow1");
- relatesTo.put("flow", relatesToIds);
+ TimelineFilterList relatesTo = new TimelineFilterList(Operator.OR);
+ Set<Object> relatesToIds =
+ new HashSet<Object>(Arrays.asList((Object)"flow1"));
+ relatesTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "flow", relatesToIds));
Set<TimelineEntity> result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null),
@@ -515,10 +782,11 @@ public class TestFileSystemTimelineReaderImpl {
}
// Get entities based on isRelatedTo.
- Map<String, Set<String>> isRelatedTo = new HashMap<String, Set<String>>();
- Set<String> isRelatedToIds = new HashSet<String>();
- isRelatedToIds.add("tid1_2");
- isRelatedTo.put("type1", isRelatedToIds);
+ TimelineFilterList isRelatedTo = new TimelineFilterList(Operator.OR);
+ Set<Object> isRelatedToIds =
+ new HashSet<Object>(Arrays.asList((Object)"tid1_2"));
+ isRelatedTo.addFilter(new TimelineKeyValuesFilter(
+ TimelineCompareOp.EQUAL, "type1", isRelatedToIds));
result = reader.getEntities(
new TimelineReaderContext("cluster1", "user1", "flow1", 1L, "app1",
"app", null),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-4460. [Bug fix] RM fails to
start when SMP is enabled. (Li Lu via Varun Saxena)
Posted by gt...@apache.org.
YARN-4460. [Bug fix] RM fails to start when SMP is enabled. (Li Lu via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4764f4ac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4764f4ac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4764f4ac
Branch: refs/heads/YARN-2928
Commit: 4764f4accc5c7fe02f892a78e979819fbf46f90c
Parents: 8d798de
Author: Varun Saxena <va...@apache.org>
Authored: Wed Dec 16 15:24:57 2015 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:17:09 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/metrics/TimelineServiceV2Publisher.java | 9 +++++----
.../metrics/TestSystemMetricsPublisherForV2.java | 5 ++++-
2 files changed, 9 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4764f4ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index b96114e..1954783 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -25,6 +25,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -75,13 +76,13 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
}
@Override
- protected void serviceStart() throws Exception {
- super.serviceStart();
+ protected void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ getDispatcher().register(SystemMetricsEventType.class,
+ new TimelineV2EventHandler());
publishContainerMetrics = getConfig().getBoolean(
YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
- getDispatcher().register(SystemMetricsEventType.class,
- new TimelineV2EventHandler());
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4764f4ac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index baaa566..57258d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -156,7 +156,7 @@ public class TestSystemMetricsPublisherForV2 {
try {
Configuration conf = getTimelineV2Conf();
conf.setBoolean(YarnConfiguration.RM_PUBLISH_CONTAINER_METRICS_ENABLED,
- false);
+ YarnConfiguration.DEFAULT_RM_PUBLISH_CONTAINER_METRICS_ENABLED);
metricsPublisher.init(conf);
assertFalse(
"Default configuration should not publish container Metrics from RM",
@@ -167,6 +167,9 @@ public class TestSystemMetricsPublisherForV2 {
metricsPublisher = new TimelineServiceV2Publisher(mock(RMContext.class));
conf = getTimelineV2Conf();
metricsPublisher.init(conf);
+ assertTrue("Expected to have registered event handlers and set ready to "
+ + "publish events after init",
+ metricsPublisher.isPublishContainerMetrics());
metricsPublisher.start();
assertTrue("Expected to publish container Metrics from RM",
metricsPublisher.isPublishContainerMetrics());
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: MAPREDUCE-6688. Store job
configurations in Timeline Service v2 (Varun Saxena via sjlee)
Posted by gt...@apache.org.
MAPREDUCE-6688. Store job configurations in Timeline Service v2 (Varun Saxena via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2c4237f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2c4237f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2c4237f
Branch: refs/heads/YARN-2928
Commit: d2c4237fa7b15aaddfdd55f906f7953b29cf2fe8
Parents: 047cde5
Author: Sangjin Lee <sj...@apache.org>
Authored: Tue May 3 09:19:36 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:38:24 2016 -0700
----------------------------------------------------------------------
.../jobhistory/JobHistoryEventHandler.java | 57 +++++++++++-
.../mapreduce/v2/app/job/impl/JobImpl.java | 2 +-
.../mapreduce/jobhistory/JobSubmittedEvent.java | 38 +++++++-
.../mapreduce/util/JobHistoryEventUtils.java | 3 +
.../mapred/TestMRTimelineEventHandling.java | 92 +++++++++++++++++---
.../org/apache/hadoop/mapred/UtilsForTests.java | 8 ++
6 files changed, 181 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index d32524e..db58ecf 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -1074,7 +1074,16 @@ public class JobHistoryEventHandler extends AbstractService
entity.setId(jobId.toString());
return entity;
}
-
+
+ private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
+ createJobEntity(JobId jobId) {
+ org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
+ new org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity();
+ entity.setId(jobId.toString());
+ entity.setType(MAPREDUCE_JOB_ENTITY_TYPE);
+ return entity;
+ }
+
// create ApplicationEntity with job finished Metrics from HistoryEvent
private org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity
createAppEntityWithJobMetrics(HistoryEvent event, JobId jobId) {
@@ -1133,6 +1142,46 @@ public class JobHistoryEventHandler extends AbstractService
return entity;
}
+ private void publishConfigsOnJobSubmittedEvent(JobSubmittedEvent event,
+ JobId jobId) {
+ if (event.getJobConf() == null) {
+ return;
+ }
+ // Publish job configurations both as job and app entity.
+ // Configs are split into multiple entities if they exceed 100kb in size.
+ org.apache.hadoop.yarn.api.records.timelineservice.
+ TimelineEntity jobEntityForConfigs = createJobEntity(jobId);
+ ApplicationEntity appEntityForConfigs = new ApplicationEntity();
+ String appId = jobId.getAppId().toString();
+ appEntityForConfigs.setId(appId);
+ try {
+ int configSize = 0;
+ for (Map.Entry<String, String> entry : event.getJobConf()) {
+ int size = entry.getKey().length() + entry.getValue().length();
+ configSize += size;
+ if (configSize > JobHistoryEventUtils.ATS_CONFIG_PUBLISH_SIZE_BYTES) {
+ if (jobEntityForConfigs.getConfigs().size() > 0) {
+ timelineClient.putEntities(jobEntityForConfigs);
+ timelineClient.putEntities(appEntityForConfigs);
+ jobEntityForConfigs = createJobEntity(jobId);
+ appEntityForConfigs = new ApplicationEntity();
+ appEntityForConfigs.setId(appId);
+ }
+ configSize = size;
+ }
+ jobEntityForConfigs.addConfig(entry.getKey(), entry.getValue());
+ appEntityForConfigs.addConfig(entry.getKey(), entry.getValue());
+ }
+ if (configSize > 0) {
+ timelineClient.putEntities(jobEntityForConfigs);
+ timelineClient.putEntities(appEntityForConfigs);
+ }
+ } catch (IOException | YarnException e) {
+ LOG.error("Exception while publishing configs on JOB_SUBMITTED Event " +
+ " for the job : " + jobId, e);
+ }
+ }
+
private void processEventForNewTimelineService(HistoryEvent event,
JobId jobId, long timestamp) {
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity tEntity =
@@ -1252,8 +1301,12 @@ public class JobHistoryEventHandler extends AbstractService
} catch (IOException | YarnException e) {
LOG.error("Failed to process Event " + event.getEventType()
+ " for the job : " + jobId, e);
+ return;
+ }
+ if (event.getEventType() == EventType.JOB_SUBMITTED) {
+ // Publish configs after main job submitted event has been posted.
+ publishConfigsOnJobSubmittedEvent((JobSubmittedEvent)event, jobId);
}
-
}
private void setSummarySlotSeconds(JobSummary summary, Counters allCounters) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index b7036a5..5127a43 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -1445,7 +1445,7 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
job.conf.get(MRJobConfig.WORKFLOW_NAME, ""),
job.conf.get(MRJobConfig.WORKFLOW_NODE_NAME, ""),
getWorkflowAdjacencies(job.conf),
- job.conf.get(MRJobConfig.WORKFLOW_TAGS, ""));
+ job.conf.get(MRJobConfig.WORKFLOW_TAGS, ""), job.conf);
job.eventHandler.handle(new JobHistoryEvent(job.jobId, jse));
//TODO JH Verify jobACLs, UserName via UGI?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
index 07edb58..7d05571 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
@@ -26,6 +26,7 @@ import java.util.Set;
import org.apache.avro.util.Utf8;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.JobACL;
import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
@InterfaceStability.Unstable
public class JobSubmittedEvent implements HistoryEvent {
private JobSubmitted datum = new JobSubmitted();
+ private JobConf jobConf = null;
/**
* Create an event to record job submission
@@ -83,6 +85,31 @@ public class JobSubmittedEvent implements HistoryEvent {
workflowAdjacencies, "");
}
+ /**
+ * Create an event to record job submission.
+ * @param id The job Id of the job
+ * @param jobName Name of the job
+ * @param userName Name of the user who submitted the job
+ * @param submitTime Time of submission
+ * @param jobConfPath Path of the Job Configuration file
+ * @param jobACLs The configured acls for the job.
+ * @param jobQueueName The job-queue to which this job was submitted to
+ * @param workflowId The Id of the workflow
+ * @param workflowName The name of the workflow
+ * @param workflowNodeName The node name of the workflow
+ * @param workflowAdjacencies The adjacencies of the workflow
+ * @param workflowTags Comma-separated tags for the workflow
+ */
+ public JobSubmittedEvent(JobID id, String jobName, String userName,
+ long submitTime, String jobConfPath,
+ Map<JobACL, AccessControlList> jobACLs, String jobQueueName,
+ String workflowId, String workflowName, String workflowNodeName,
+ String workflowAdjacencies, String workflowTags) {
+ this(id, jobName, userName, submitTime, jobConfPath, jobACLs,
+ jobQueueName, workflowId, workflowName, workflowNodeName,
+ workflowAdjacencies, workflowTags, null);
+ }
+
/**
* Create an event to record job submission
* @param id The job Id of the job
@@ -97,12 +124,13 @@ public class JobSubmittedEvent implements HistoryEvent {
* @param workflowNodeName The node name of the workflow
* @param workflowAdjacencies The adjacencies of the workflow
* @param workflowTags Comma-separated tags for the workflow
+ * @param conf Job configuration
*/
public JobSubmittedEvent(JobID id, String jobName, String userName,
long submitTime, String jobConfPath,
Map<JobACL, AccessControlList> jobACLs, String jobQueueName,
String workflowId, String workflowName, String workflowNodeName,
- String workflowAdjacencies, String workflowTags) {
+ String workflowAdjacencies, String workflowTags, JobConf conf) {
datum.setJobid(new Utf8(id.toString()));
datum.setJobName(new Utf8(jobName));
datum.setUserName(new Utf8(userName));
@@ -132,6 +160,7 @@ public class JobSubmittedEvent implements HistoryEvent {
if (workflowTags != null) {
datum.setWorkflowTags(new Utf8(workflowTags));
}
+ jobConf = conf;
}
JobSubmittedEvent() {}
@@ -208,7 +237,11 @@ public class JobSubmittedEvent implements HistoryEvent {
}
/** Get the event type */
public EventType getEventType() { return EventType.JOB_SUBMITTED; }
-
+
+ public JobConf getJobConf() {
+ return jobConf;
+ }
+
@Override
public TimelineEvent toTimelineEvent() {
TimelineEvent tEvent = new TimelineEvent();
@@ -234,5 +267,4 @@ public class JobSubmittedEvent implements HistoryEvent {
public Set<TimelineMetric> getTimelineMetrics() {
return null;
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
index 225d517..35d066c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/JobHistoryEventUtils.java
@@ -36,6 +36,9 @@ public final class JobHistoryEventUtils {
private JobHistoryEventUtils() {
}
+ // Number of bytes of config which can be published in one shot to ATSv2.
+ public static final int ATS_CONFIG_PUBLISH_SIZE_BYTES = 10 * 1024;
+
public static JsonNode countersToJSON(Counters counters) {
ObjectMapper mapper = new ObjectMapper();
ArrayNode nodes = mapper.createArrayNode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 5915a43..fde9e64 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -26,7 +26,9 @@ import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.EnumSet;
+import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
@@ -55,6 +57,8 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.collect.Sets;
+
public class TestMRTimelineEventHandling {
private static final String TIMELINE_AUX_SERVICE_NAME = "timeline_collector";
@@ -191,8 +195,17 @@ public class TestMRTimelineEventHandling {
Path inDir = new Path("input");
Path outDir = new Path("output");
LOG.info("Run 1st job which should be successful.");
+ JobConf successConf = new JobConf(conf);
+ successConf.set("dummy_conf1",
+ UtilsForTests.createConfigValue(51 * 1024));
+ successConf.set("dummy_conf2",
+ UtilsForTests.createConfigValue(51 * 1024));
+ successConf.set("huge_dummy_conf1",
+ UtilsForTests.createConfigValue(101 * 1024));
+ successConf.set("huge_dummy_conf2",
+ UtilsForTests.createConfigValue(101 * 1024));
RunningJob job =
- UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
+ UtilsForTests.runJobSucceed(successConf, inDir, outDir);
Assert.assertEquals(JobStatus.SUCCEEDED,
job.getJobStatus().getState().getValue());
@@ -270,7 +283,11 @@ public class TestMRTimelineEventHandling {
Assert.assertTrue("jobEventFilePath: " + jobEventFilePath +
" does not exist.",
jobEventFile.exists());
- verifyMetricsWhenEvent(jobEventFile, EventType.JOB_FINISHED.name());
+ verifyEntity(jobEventFile, EventType.JOB_FINISHED.name(),
+ true, false, null);
+ Set<String> cfgsToCheck = Sets.newHashSet("dummy_conf1", "dummy_conf2",
+ "huge_dummy_conf1", "huge_dummy_conf2");
+ verifyEntity(jobEventFile, null, false, true, cfgsToCheck);
// for this test, we expect MR job metrics are published in YARN_APPLICATION
String outputAppDir = basePath + "/YARN_APPLICATION/";
@@ -290,7 +307,8 @@ public class TestMRTimelineEventHandling {
"appEventFilePath: " + appEventFilePath +
" does not exist.",
appEventFile.exists());
- verifyMetricsWhenEvent(appEventFile, null);
+ verifyEntity(appEventFile, null, true, false, null);
+ verifyEntity(appEventFile, null, false, true, cfgsToCheck);
// check for task event file
String outputDirTask = basePath + "/MAPREDUCE_TASK/";
@@ -307,7 +325,8 @@ public class TestMRTimelineEventHandling {
Assert.assertTrue("taskEventFileName: " + taskEventFilePath +
" does not exist.",
taskEventFile.exists());
- verifyMetricsWhenEvent(taskEventFile, EventType.TASK_FINISHED.name());
+ verifyEntity(taskEventFile, EventType.TASK_FINISHED.name(),
+ true, false, null);
// check for task attempt event file
String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/";
@@ -324,17 +343,30 @@ public class TestMRTimelineEventHandling {
File taskAttemptEventFile = new File(taskAttemptEventFilePath);
Assert.assertTrue("taskAttemptEventFileName: " + taskAttemptEventFilePath +
" does not exist.", taskAttemptEventFile.exists());
- verifyMetricsWhenEvent(taskAttemptEventFile,
- EventType.MAP_ATTEMPT_FINISHED.name());
+ verifyEntity(taskAttemptEventFile, EventType.MAP_ATTEMPT_FINISHED.name(),
+ true, false, null);
}
- private void verifyMetricsWhenEvent(File entityFile, String eventId)
+ /**
+ * Verifies entity by reading the entity file written via FS impl.
+ * @param entityFile File to be read.
+ * @param eventId Event to be checked.
+ * @param chkMetrics If event is not null, this flag determines if metrics
+ * exist when the event is encountered. If event is null, we merely check
+ * if metrics exist in the entity file.
+ * @param chkCfg If event is not null, this flag determines if configs
+ * exist when the event is encountered. If event is null, we merely check
+ * if configs exist in the entity file.
+ * @param cfgsToVerify a set of configs which should exist in the entity file.
+ * @throws IOException
+ */
+ private void verifyEntity(File entityFile, String eventId,
+ boolean chkMetrics, boolean chkCfg, Set<String> cfgsToVerify)
throws IOException {
BufferedReader reader = null;
String strLine;
try {
reader = new BufferedReader(new FileReader(entityFile));
- boolean jobMetricsFoundForAppEntity = false;
while ((strLine = reader.readLine()) != null) {
if (strLine.trim().length() > 0) {
org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity entity =
@@ -344,23 +376,57 @@ public class TestMRTimelineEventHandling {
if (eventId == null) {
// Job metrics are published without any events for
// ApplicationEntity. There is also possibility that some other
- // ApplicationEntity is published without events, hence loop all
- if (entity.getEvents().size() == 0) {
- jobMetricsFoundForAppEntity = entity.getMetrics().size() > 0;
- if (jobMetricsFoundForAppEntity) {
+ // ApplicationEntity is published without events, hence loop till
+ // its found. Same applies to configs.
+ if (chkMetrics && entity.getMetrics().size() > 0) {
+ return;
+ }
+ if (chkCfg && entity.getConfigs().size() > 0) {
+ if (cfgsToVerify == null) {
return;
+ } else {
+ // Have configs to verify. Keep on removing configs from the set
+ // of configs to verify as they are found. When the all the
+ // entities have been looped through, we will check if the set
+ // is empty or not(indicating if all configs have been found or
+ // not).
+ for (Iterator<String> itr =
+ cfgsToVerify.iterator(); itr.hasNext();) {
+ String config = itr.next();
+ if (entity.getConfigs().containsKey(config)) {
+ itr.remove();
+ }
+ }
+ // All the required configs have been verified, so return.
+ if (cfgsToVerify.isEmpty()) {
+ return;
+ }
}
}
} else {
for (TimelineEvent event : entity.getEvents()) {
if (event.getId().equals(eventId)) {
- assertTrue(entity.getMetrics().size() > 0);
+ if (chkMetrics) {
+ assertTrue(entity.getMetrics().size() > 0);
+ }
+ if (chkCfg) {
+ assertTrue(entity.getConfigs().size() > 0);
+ if (cfgsToVerify != null) {
+ for (String cfg : cfgsToVerify) {
+ assertTrue(entity.getConfigs().containsKey(cfg));
+ }
+ }
+ }
return;
}
}
}
}
}
+ if (cfgsToVerify != null) {
+ assertTrue(cfgsToVerify.isEmpty());
+ return;
+ }
fail("Expected event : " + eventId + " not found in the file "
+ entityFile);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2c4237f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
index cb494db..2fb6828 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/UtilsForTests.java
@@ -156,6 +156,14 @@ public class UtilsForTests {
return buf.toString();
}
+ public static String createConfigValue(int msgSize) {
+ StringBuilder sb = new StringBuilder(msgSize);
+ for (int i=0; i<msgSize; i++) {
+ sb.append('a');
+ }
+ return sb.toString();
+ }
+
public static String safeGetCanonicalPath(File f) {
try {
String s = f.getCanonicalPath();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: YARN-4409. Fix javadoc and
checkstyle issues in timelineservice code (Varun Saxena via sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index fc05310..12daa95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -388,15 +388,15 @@ public class RMActiveServiceContext {
@Private
@Unstable
public void setRMTimelineCollectorManager(
- RMTimelineCollectorManager timelineCollectorManager) {
- this.timelineCollectorManager = timelineCollectorManager;
+ RMTimelineCollectorManager collectorManager) {
+ this.timelineCollectorManager = collectorManager;
}
@Private
@Unstable
public void setSystemMetricsPublisher(
- SystemMetricsPublisher systemMetricsPublisher) {
- this.systemMetricsPublisher = systemMetricsPublisher;
+ SystemMetricsPublisher metricsPublisher) {
+ this.systemMetricsPublisher = metricsPublisher;
}
@Private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index e122ab4..4c72912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -379,8 +379,8 @@ public class RMContextImpl implements RMContext {
@Override
public void setSystemMetricsPublisher(
- SystemMetricsPublisher systemMetricsPublisher) {
- this.systemMetricsPublisher = systemMetricsPublisher;
+ SystemMetricsPublisher metricsPublisher) {
+ this.systemMetricsPublisher = metricsPublisher;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
index a8c00a4..d4a4fc3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AbstractSystemMetricsPublisher.java
@@ -30,6 +30,10 @@ import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.Event;
import org.apache.hadoop.yarn.event.EventHandler;
+/**
+ * Abstract implementation of SystemMetricsPublisher which is then extended by
+ * metrics publisher implementations depending on timeline service version.
+ */
public abstract class AbstractSystemMetricsPublisher extends CompositeService
implements SystemMetricsPublisher {
private MultiThreadedDispatcher dispatcher;
@@ -46,13 +50,18 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService
protected void serviceInit(Configuration conf) throws Exception {
dispatcher =
new MultiThreadedDispatcher(getConfig().getInt(
- YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
- YarnConfiguration.DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE));
+ YarnConfiguration.
+ RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE,
+ YarnConfiguration.
+ DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE));
dispatcher.setDrainEventsOnStop();
addIfService(dispatcher);
super.serviceInit(conf);
}
+ /**
+ * Dispatches ATS related events using multiple threads.
+ */
@SuppressWarnings({ "rawtypes", "unchecked" })
public static class MultiThreadedDispatcher extends CompositeService
implements Dispatcher {
@@ -107,7 +116,7 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService
}
/**
- * EventType which is used while publishing the events
+ * EventType which is used while publishing the events.
*/
protected static enum SystemMetricsEventType {
PUBLISH_ENTITY, PUBLISH_APPLICATION_FINISHED_ENTITY
@@ -158,9 +167,10 @@ public abstract class AbstractSystemMetricsPublisher extends CompositeService
if (other.getType() != null) {
return false;
}
- } else
+ } else {
if (!appId.equals(other.appId) || !getType().equals(other.getType())) {
- return false;
+ return false;
+ }
}
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
index 1810df1..c0c066a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/NoOpSystemMetricPublisher.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
/**
* This class does nothing when any of the methods are invoked on
- * SystemMetricsPublisher
+ * SystemMetricsPublisher.
*/
public class NoOpSystemMetricPublisher implements SystemMetricsPublisher{
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index f895bba..8e8acbc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+/**
+ * Interface used to publish app/container events to timelineservice.
+ */
public interface SystemMetricsPublisher {
void appCreated(RMApp app, long createdTime);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
index ddc8a16..7dc5073 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV1Publisher.java
@@ -43,6 +43,10 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+/**
+ * This class is responsible for posting application, appattempt & Container
+ * lifecycle related events to timeline service v1.
+ */
public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
private static final Log LOG =
@@ -137,8 +141,8 @@ public class TimelineServiceV1Publisher extends AbstractSystemMetricsPublisher {
entity.addEvent(tEvent);
// sync sending of finish event to avoid possibility of saving application
- // finished state in RMStateStore save without publishing in ATS
- putEntity(entity);// sync event so that ATS update is done without fail
+ // finished state in RMStateStore save without publishing in ATS.
+ putEntity(entity); // sync event so that ATS update is done without fail.
}
@SuppressWarnings("unchecked")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
index 1b3aa6f..14073d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TimelineServiceV2Publisher.java
@@ -60,14 +60,14 @@ import com.google.common.annotations.VisibleForTesting;
/**
* This class is responsible for posting application, appattempt & Container
- * lifecycle related events to timeline service V2
+ * lifecycle related events to timeline service v2.
*/
@Private
@Unstable
public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
private static final Log LOG =
LogFactory.getLog(TimelineServiceV2Publisher.class);
- protected RMTimelineCollectorManager rmTimelineCollectorManager;
+ private RMTimelineCollectorManager rmTimelineCollectorManager;
private boolean publishContainerMetrics;
public TimelineServiceV2Publisher(RMContext rmContext) {
@@ -342,7 +342,8 @@ public class TimelineServiceV2Publisher extends AbstractSystemMetricsPublisher {
}
}
- private static ContainerEntity createContainerEntity(ContainerId containerId) {
+ private static ContainerEntity createContainerEntity(
+ ContainerId containerId) {
ContainerEntity entity = new ContainerEntity();
entity.setId(containerId.toString());
entity.setParent(new Identifier(TimelineEntityType.YARN_APPLICATION_ATTEMPT
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java
new file mode 100644
index 0000000..a8a3804
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.resourcemanager.metrics contains
+ * classes related to publishing app/container events to ATS.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.metrics;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
index 698c9b5..9642911 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppCollectorUpdateEvent.java
@@ -20,6 +20,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+/**
+ * Event used for updating collector address in RMApp on node heartbeat.
+ */
public class RMAppCollectorUpdateEvent extends RMAppEvent {
private final String appCollectorAddr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index cea7563..b0eccc4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -98,7 +98,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.timelineservice.collector.AppLevelTimelineCollector;
-import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
@@ -614,8 +613,8 @@ public class RMAppImpl implements RMApp, Recoverable {
}
@Override
- public void setCollectorAddr(String collectorAddr) {
- this.collectorAddr = collectorAddr;
+ public void setCollectorAddr(String collectorAddress) {
+ this.collectorAddr = collectorAddress;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
index 116bf64..ff055a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -28,7 +28,10 @@ import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-
+/**
+ * This class extends TimelineCollectorManager to provide RM specific
+ * implementations.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class RMTimelineCollectorManager extends TimelineCollectorManager {
@@ -44,8 +47,8 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
RMApp app = rmContext.getRMApps().get(appId);
if (app == null) {
throw new YarnRuntimeException(
- "Unable to get the timeline collector context info for a non-existing app " +
- appId);
+ "Unable to get the timeline collector context info for a " +
+ "non-existing app " + appId);
}
String userId = app.getUser();
if (userId != null && !userId.isEmpty()) {
@@ -57,18 +60,18 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
continue;
}
switch (parts[0].toUpperCase()) {
- case TimelineUtils.FLOW_NAME_TAG_PREFIX:
- collector.getTimelineEntityContext().setFlowName(parts[1]);
- break;
- case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
- collector.getTimelineEntityContext().setFlowVersion(parts[1]);
- break;
- case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
- collector.getTimelineEntityContext().setFlowRunId(
- Long.parseLong(parts[1]));
- break;
- default:
- break;
+ case TimelineUtils.FLOW_NAME_TAG_PREFIX:
+ collector.getTimelineEntityContext().setFlowName(parts[1]);
+ break;
+ case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
+ collector.getTimelineEntityContext().setFlowVersion(parts[1]);
+ break;
+ case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
+ collector.getTimelineEntityContext().setFlowRunId(
+ Long.parseLong(parts[1]));
+ break;
+ default:
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java
new file mode 100644
index 0000000..c470011
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/package-info.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.resourcemanager.timelineservice
+ * contains classes related to handling of app level collectors.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.resourcemanager.timelineservice;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index b148491..36dd7b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -55,7 +55,8 @@ public class AppLevelTimelineCollector extends TimelineCollector {
// Current user usually is not the app user, but keep this field non-null
context.setUserId(UserGroupInformation.getCurrentUser().getShortUserName());
// Use app ID to generate a default flow name for orphan app
- context.setFlowName(TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId));
+ context.setFlowName(
+ TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId));
// Set the flow version to string 1 if it's an orphan app
context.setFlowVersion("1");
// Set the flow run ID to 1 if it's an orphan app
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 0977ed9..785fb19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -48,13 +48,17 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
+/**
+ * Class on the NodeManager side that manages adding and removing collectors and
+ * their lifecycle. Also instantiates the per-node collector webapp.
+ */
@Private
@Unstable
public class NodeTimelineCollectorManager extends TimelineCollectorManager {
private static final Log LOG =
LogFactory.getLog(NodeTimelineCollectorManager.class);
- // REST server for this collector manager
+ // REST server for this collector manager.
private HttpServer2 timelineRestServer;
private String timelineRestServerBindAddress;
@@ -97,7 +101,7 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
}
/**
- * Launch the REST web server for this collector manager
+ * Launch the REST web server for this collector manager.
*/
private void startWebApp() {
Configuration conf = getConfig();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
index b738530..041e7c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/PerNodeTimelineCollectorsAuxService.java
@@ -108,6 +108,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
* The collector is also initialized and started. If the service already
* exists, no new service is created.
*
+ * @param appId Application Id to be added.
* @return whether it was added successfully
*/
public boolean addApplication(ApplicationId appId) {
@@ -122,6 +123,7 @@ public class PerNodeTimelineCollectorsAuxService extends AuxiliaryService {
* collector is also stopped as a result. If the collector does not exist, no
* change is made.
*
+ * @param appId Application Id to be removed.
* @return whether it was removed successfully
*/
public boolean removeApplication(ApplicationId appId) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
index bb7db12..15187d1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollector.java
@@ -80,6 +80,8 @@ public abstract class TimelineCollector extends CompositeService {
* @param entities entities to post
* @param callerUgi the caller UGI
* @return the response that contains the result of the post.
+ * @throws IOException if there is any exception encountered while putting
+ * entities.
*/
public TimelineWriteResponse putEntities(TimelineEntities entities,
UserGroupInformation callerUgi) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
index 165754d..8f74ffb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorManager.java
@@ -68,8 +68,10 @@ public class TimelineCollectorManager extends AbstractService {
// basis
writerFlusher = Executors.newSingleThreadScheduledExecutor();
flushInterval = conf.getInt(
- YarnConfiguration.TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS,
- YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS);
+ YarnConfiguration.
+ TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS,
+ YarnConfiguration.
+ DEFAULT_TIMELINE_SERVICE_WRITER_FLUSH_INTERVAL_SECONDS);
super.serviceInit(conf);
}
@@ -102,6 +104,8 @@ public class TimelineCollectorManager extends AbstractService {
* Put the collector into the collection if an collector mapped by id does
* not exist.
*
+ * @param appId Application Id for which collector needs to be put.
+ * @param collector timeline collector to be put.
* @throws YarnRuntimeException if there was any exception in initializing
* and starting the app level service
* @return the collector associated with id after the potential put.
@@ -140,6 +144,7 @@ public class TimelineCollectorManager extends AbstractService {
* Removes the collector for the specified id. The collector is also stopped
* as a result. If the collector does not exist, no change is made.
*
+ * @param appId Application Id to remove.
* @return whether it was removed successfully
*/
public boolean remove(ApplicationId appId) {
@@ -162,6 +167,7 @@ public class TimelineCollectorManager extends AbstractService {
/**
* Returns the collector for the specified id.
*
+ * @param appId Application Id for which we need to get the collector.
* @return the collector or null if it does not exist
*/
public TimelineCollector get(ApplicationId appId) {
@@ -171,6 +177,8 @@ public class TimelineCollectorManager extends AbstractService {
/**
* Returns whether the collector for the specified id exists in this
* collection.
+ * @param appId Application Id.
+ * @return true if collector for the app id is found, false otherwise.
*/
public boolean containsTimelineCollector(ApplicationId appId) {
return collectors.containsKey(appId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
index 8f595e2..2dff937 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorWebService.java
@@ -74,6 +74,9 @@ public class TimelineCollectorWebService {
private @Context ServletContext context;
+ /**
+ * Gives information about timeline collector.
+ */
@XmlRootElement(name = "about")
@XmlAccessorType(XmlAccessType.NONE)
@Public
@@ -86,8 +89,8 @@ public class TimelineCollectorWebService {
}
- public AboutInfo(String about) {
- this.about = about;
+ public AboutInfo(String abt) {
+ this.about = abt;
}
@XmlElement(name = "About")
@@ -95,14 +98,18 @@ public class TimelineCollectorWebService {
return about;
}
- public void setAbout(String about) {
- this.about = about;
+ public void setAbout(String abt) {
+ this.about = abt;
}
}
/**
* Return the description of the timeline web services.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @return description of timeline web service.
*/
@GET
@Produces({ MediaType.APPLICATION_JSON /* , MediaType.APPLICATION_XML */})
@@ -117,6 +124,15 @@ public class TimelineCollectorWebService {
* Accepts writes to the collector, and returns a response. It simply routes
* the request to the app level collector. It expects an application as a
* context.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param async flag indicating whether its an async put or not. "true"
+ * indicates, its an async call. If null, its considered false.
+ * @param appId Application Id to which the entities to be put belong to. If
+ * appId is not there or it cannot be parsed, HTTP 400 will be sent back.
+ * @param entities timeline entities to be put.
+ * @return a Response with appropriate HTTP status.
*/
@PUT
@Path("/entities")
@@ -202,29 +218,29 @@ public class TimelineCollectorWebService {
}
if (type != null) {
switch (type) {
- case YARN_CLUSTER:
- entitiesToReturn.addEntity(new ClusterEntity(entity));
- break;
- case YARN_FLOW_RUN:
- entitiesToReturn.addEntity(new FlowRunEntity(entity));
- break;
- case YARN_APPLICATION:
- entitiesToReturn.addEntity(new ApplicationEntity(entity));
- break;
- case YARN_APPLICATION_ATTEMPT:
- entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity));
- break;
- case YARN_CONTAINER:
- entitiesToReturn.addEntity(new ContainerEntity(entity));
- break;
- case YARN_QUEUE:
- entitiesToReturn.addEntity(new QueueEntity(entity));
- break;
- case YARN_USER:
- entitiesToReturn.addEntity(new UserEntity(entity));
- break;
- default:
- break;
+ case YARN_CLUSTER:
+ entitiesToReturn.addEntity(new ClusterEntity(entity));
+ break;
+ case YARN_FLOW_RUN:
+ entitiesToReturn.addEntity(new FlowRunEntity(entity));
+ break;
+ case YARN_APPLICATION:
+ entitiesToReturn.addEntity(new ApplicationEntity(entity));
+ break;
+ case YARN_APPLICATION_ATTEMPT:
+ entitiesToReturn.addEntity(new ApplicationAttemptEntity(entity));
+ break;
+ case YARN_CONTAINER:
+ entitiesToReturn.addEntity(new ContainerEntity(entity));
+ break;
+ case YARN_QUEUE:
+ entitiesToReturn.addEntity(new QueueEntity(entity));
+ break;
+ case YARN_USER:
+ entitiesToReturn.addEntity(new UserEntity(entity));
+ break;
+ default:
+ break;
}
} else {
entitiesToReturn.addEntity(entity);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java
new file mode 100644
index 0000000..1f7dd23
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/package-info.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.collector contains
+ * classes which can be used across collector. This package contains classes
+ * which are not related to storage implementations though.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.server.timelineservice.collector;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
index 84a97ea..4cff3bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderManager.java
@@ -34,6 +34,11 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader;
import com.google.common.annotations.VisibleForTesting;
+/**
+ * This class wraps over the timeline reader store implementation. It does some
+ * non trivial manipulation of the timeline data before or after getting
+ * it from the backend store.
+ */
@Private
@Unstable
public class TimelineReaderManager extends AbstractService {
@@ -114,9 +119,19 @@ public class TimelineReaderManager extends AbstractService {
}
/**
- * Get a set of entities matching given predicates. The meaning of each
- * argument has been documented with {@link TimelineReader#getEntities}.
+ * Get a set of entities matching given predicates by making a call to
+ * backend storage implementation. The meaning of each argument has been
+ * documented in detail with {@link TimelineReader#getEntities}.If cluster ID
+ * has not been supplied by the client, fills the cluster id from config
+ * before making a call to backend storage. After fetching entities from
+ * backend, fills the appropriate UID based on entity type for each entity.
*
+ * @param context Timeline context within the scope of which entities have to
+ * be fetched.
+ * @param filters Filters which limit the number of entities to be returned.
+ * @param dataToRetrieve Data to carry in each entity fetched.
+ * @return a set of <cite>TimelineEntity</cite> objects.
+ * @throws IOException if any problem occurs while getting entities.
* @see TimelineReader#getEntities
*/
public Set<TimelineEntity> getEntities(TimelineReaderContext context,
@@ -135,9 +150,18 @@ public class TimelineReaderManager extends AbstractService {
}
/**
- * Get single timeline entity. The meaning of each argument has been
- * documented with {@link TimelineReader#getEntity}.
+ * Get single timeline entity by making a call to backend storage
+ * implementation. The meaning of each argument in detail has been
+ * documented with {@link TimelineReader#getEntity}. If cluster ID has not
+ * been supplied by the client, fills the cluster id from config before making
+ * a call to backend storage. After fetching entity from backend, fills the
+ * appropriate UID based on entity type.
*
+ * @param context Timeline context within the scope of which entity has to be
+ * fetched.
+ * @param dataToRetrieve Data to carry in the entity fetched.
+ * @return A <cite>TimelineEntity</cite> object if found, null otherwise.
+ * @throws IOException if any problem occurs while getting entity.
* @see TimelineReader#getEntity
*/
public TimelineEntity getEntity(TimelineReaderContext context,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
index 6b4213d..97725e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderServer.java
@@ -49,7 +49,7 @@ import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import com.google.common.annotations.VisibleForTesting;
-/** Main class for Timeline Reader */
+/** Main class for Timeline Reader. */
@Private
@Unstable
public class TimelineReaderServer extends CompositeService {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index d68199a..0b9549b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -73,24 +73,25 @@ public class TimelineReaderWebServices {
private static final String DATE_PATTERN = "yyyyMMdd";
@VisibleForTesting
- static ThreadLocal<DateFormat> DATE_FORMAT = new ThreadLocal<DateFormat>() {
- @Override
- protected DateFormat initialValue() {
- SimpleDateFormat format =
- new SimpleDateFormat(DATE_PATTERN, Locale.ENGLISH);
- format.setTimeZone(TimeZone.getTimeZone("GMT"));
- format.setLenient(false);
- return format;
- }
- };
+ static final ThreadLocal<DateFormat> DATE_FORMAT =
+ new ThreadLocal<DateFormat>() {
+ @Override
+ protected DateFormat initialValue() {
+ SimpleDateFormat format =
+ new SimpleDateFormat(DATE_PATTERN, Locale.ENGLISH);
+ format.setTimeZone(TimeZone.getTimeZone("GMT"));
+ format.setLenient(false);
+ return format;
+ }
+ };
private void init(HttpServletResponse response) {
response.setContentType(null);
}
- private static class DateRange {
- Long dateStart;
- Long dateEnd;
+ private static final class DateRange {
+ private Long dateStart;
+ private Long dateEnd;
private DateRange(Long start, Long end) {
this.dateStart = start;
this.dateEnd = end;
@@ -212,6 +213,7 @@ public class TimelineReaderWebServices {
* @param uId a delimited string containing clusterid, userid, flow name,
* flowrun id and app id which are extracted from UID and then used to
* query backend(Mandatory path param).
+ * @param entityType Type of entities(Mandatory path param).
* @param limit Number of entities to return(Optional query param).
* @param createdTimeStart If specified, matched entities should not be
* created before this timestamp(Optional query param).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
index fc07e51..d12f7e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -44,15 +44,15 @@ final class TimelineReaderWebServicesUtils {
/**
* Parse the passed context information represented as strings and convert
* into a {@link TimelineReaderContext} object.
- * @param clusterId
- * @param userId
- * @param flowName
- * @param flowRunId
- * @param appId
- * @param entityType
- * @param entityId
+ * @param clusterId Cluster Id.
+ * @param userId User Id.
+ * @param flowName Flow Name.
+ * @param flowRunId Run id for the flow.
+ * @param appId App Id.
+ * @param entityType Entity Type.
+ * @param entityId Entity Id.
* @return a {@link TimelineReaderContext} object.
- * @throws Exception
+ * @throws Exception if any problem occurs during parsing.
*/
static TimelineReaderContext createTimelineReaderContext(String clusterId,
String userId, String flowName, String flowRunId, String appId,
@@ -65,17 +65,17 @@ final class TimelineReaderWebServicesUtils {
/**
* Parse the passed filters represented as strings and convert them into a
* {@link TimelineEntityFilters} object.
- * @param limit
- * @param createdTimeStart
- * @param createdTimeEnd
- * @param relatesTo
- * @param isRelatedTo
- * @param infofilters
- * @param conffilters
- * @param metricfilters
- * @param eventfilters
+ * @param limit Limit to number of entities to return.
+ * @param createdTimeStart Created time start for the entities to return.
+ * @param createdTimeEnd Created time end for the entities to return.
+ * @param relatesTo Entities to return must match relatesTo.
+ * @param isRelatedTo Entities to return must match isRelatedTo.
+ * @param infofilters Entities to return must match these info filters.
+ * @param conffilters Entities to return must match these metric filters.
+ * @param metricfilters Entities to return must match these metric filters.
+ * @param eventfilters Entities to return must match these event filters.
* @return a {@link TimelineEntityFilters} object.
- * @throws Exception
+ * @throws Exception if any problem occurs during parsing.
*/
static TimelineEntityFilters createTimelineEntityFilters(String limit,
String createdTimeStart, String createdTimeEnd, String relatesTo,
@@ -94,11 +94,11 @@ final class TimelineReaderWebServicesUtils {
/**
* Parse the passed fields represented as strings and convert them into a
* {@link TimelineDataToRetrieve} object.
- * @param confs
- * @param metrics
- * @param fields
+ * @param confs confs to retrieve.
+ * @param metrics metrics to retrieve.
+ * @param fields fields to retrieve.
* @return a {@link TimelineDataToRetrieve} object.
- * @throws Exception
+ * @throws Exception if any problem occurs during parsing.
*/
static TimelineDataToRetrieve createTimelineDataToRetrieve(String confs,
String metrics, String fields) throws Exception {
@@ -192,7 +192,7 @@ final class TimelineReaderWebServicesUtils {
* should be represented as "key1:value1,key2:value2,key3:value3".
* @param str delimited string represented as key-value pairs.
* @param pairsDelim key-value pairs are delimited by this delimiter.
- * @param keyValuesDelim key and value are delimited by this delimiter.
+ * @param keyValDelim key and value are delimited by this delimiter.
* @return a map of key-value pairs with both key and value being strings.
*/
static Map<String, String> parseKeyStrValueStr(String str,
@@ -212,8 +212,8 @@ final class TimelineReaderWebServicesUtils {
* should be represented as "key1:value1,key2:value2,key3:value3".
* @param str delimited string represented as key-value pairs.
* @param pairsDelim key-value pairs are delimited by this delimiter.
- * @param keyValuesDelim key and value are delimited by this delimiter.
- * @return a map of key-value pairs with key being a string and value amy
+ * @param keyValDelim key and value are delimited by this delimiter.
+ * @return a map of key-value pairs with key being a string and value, any
* object.
*/
static Map<String, Object> parseKeyStrValueObj(String str,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
index cac3f06..08e5405 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
@@ -204,7 +204,7 @@ enum TimelineUIDConverter {
/**
* Split UID using {@link #UID_DELIMITER_CHAR} and {@link #UID_ESCAPE_CHAR}.
- * @param uid
+ * @param uid UID to be splitted.
* @return a list of different parts of UID split across delimiter.
* @throws IllegalArgumentException if UID is not properly escaped.
*/
@@ -229,17 +229,19 @@ enum TimelineUIDConverter {
/**
* Encodes UID depending on UID implementation.
- * @param context
+ *
+ * @param context Reader context.
* @return UID represented as a string.
*/
abstract String encodeUID(TimelineReaderContext context);
/**
* Decodes UID depending on UID implementation.
- * @param uId
+ *
+ * @param uId UID to be decoded.
* @return a {@link TimelineReaderContext} object if UID passed can be
* decoded, null otherwise.
- * @throws Exception
+ * @throws Exception if any problem occurs while decoding.
*/
abstract TimelineReaderContext decodeUID(String uId) throws Exception;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
index da3c383..f902500 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/filter/TimelineFilterUtils.java
@@ -93,9 +93,11 @@ public final class TimelineFilterUtils {
* Creates equivalent HBase {@link FilterList} from {@link TimelineFilterList}
* while converting different timeline filters(of type {@link TimelineFilter})
* into their equivalent HBase filters.
- * @param colPrefix
- * @param filterList
- * @return a {@link FilterList} object
+ *
+ * @param <T> Describes the type of column prefix.
+ * @param colPrefix column prefix which will be used for conversion.
+ * @param filterList timeline filter list which has to be converted.
+ * @return A {@link FilterList} object.
*/
public static <T> FilterList createHBaseFilterList(ColumnPrefix<T> colPrefix,
TimelineFilterList filterList) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
index 51247bd..116509a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/package-info.java
@@ -15,6 +15,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
+/**
+ * Package org.apache.hadoop.yarn.server.timelineservice.reader contains classes
+ * which can be used across reader. This package contains classes which are
+ * not related to storage implementations.
+ */
@InterfaceAudience.Private
@InterfaceStability.Unstable
package org.apache.hadoop.yarn.server.timelineservice.reader;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
index aa0071f..97e05dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineReaderImpl.java
@@ -105,15 +105,16 @@ public class FileSystemTimelineReaderImpl extends AbstractService
/**
* Deserialize a POJO object from a JSON string.
- * @param clazz
- * class to be desirialized
*
- * @param jsonString
- * json string to deserialize
- * @return TimelineEntity object
- * @throws IOException
- * @throws JsonMappingException
- * @throws JsonGenerationException
+ * @param <T> Describes the type of class to be returned.
+ * @param clazz class to be deserialized.
+ * @param jsonString JSON string to deserialize.
+ * @return An object based on class type. Used typically for
+ * <cite>TimelineEntity</cite> object.
+ * @throws IOException if the underlying input source has problems during
+ * parsing.
+ * @throws JsonMappingException if parser has problems parsing content.
+ * @throws JsonGenerationException if there is a problem in JSON writing.
*/
public static <T> T getTimelineRecordFromJSON(
String jsonString, Class<T> clazz)
@@ -128,33 +129,32 @@ public class FileSystemTimelineReaderImpl extends AbstractService
}
for (Field field : fields) {
switch(field) {
- case CONFIGS:
- finalEntity.setConfigs(real.getConfigs());
- break;
- case METRICS:
- finalEntity.setMetrics(real.getMetrics());
- break;
- case INFO:
- finalEntity.setInfo(real.getInfo());
- break;
- case IS_RELATED_TO:
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- break;
- case RELATES_TO:
- finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
- break;
- case EVENTS:
- finalEntity.setEvents(real.getEvents());
- break;
- default:
- continue;
+ case CONFIGS:
+ finalEntity.setConfigs(real.getConfigs());
+ break;
+ case METRICS:
+ finalEntity.setMetrics(real.getMetrics());
+ break;
+ case INFO:
+ finalEntity.setInfo(real.getInfo());
+ break;
+ case IS_RELATED_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case RELATES_TO:
+ finalEntity.setIsRelatedToEntities(real.getIsRelatedToEntities());
+ break;
+ case EVENTS:
+ finalEntity.setEvents(real.getEvents());
+ break;
+ default:
+ continue;
}
}
}
- private String getFlowRunPath(String userId, String clusterId, String flowName,
- Long flowRunId, String appId)
- throws IOException {
+ private String getFlowRunPath(String userId, String clusterId,
+ String flowName, Long flowRunId, String appId) throws IOException {
if (userId != null && flowName != null && flowRunId != null) {
return userId + "/" + flowName + "/" + flowRunId;
}
@@ -272,11 +272,11 @@ public class FileSystemTimelineReaderImpl extends AbstractService
Map<Long, Set<TimelineEntity>> sortedEntities =
new TreeMap<>(
new Comparator<Long>() {
- @Override
- public int compare(Long l1, Long l2) {
- return l2.compareTo(l1);
- }
+ @Override
+ public int compare(Long l1, Long l2) {
+ return l2.compareTo(l1);
}
+ }
);
for (File entityFile : dir.listFiles()) {
if (!entityFile.getName().contains(TIMELINE_SERVICE_STORAGE_EXTENSION)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
index 4385bbc..74a03ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FileSystemTimelineWriterImpl.java
@@ -76,9 +76,10 @@ public class FileSystemTimelineWriterImpl extends AbstractService
return response;
}
- private synchronized void write(String clusterId, String userId, String flowName,
- String flowVersion, long flowRun, String appId, TimelineEntity entity,
- TimelineWriteResponse response) throws IOException {
+ private synchronized void write(String clusterId, String userId,
+ String flowName, String flowVersion, long flowRun, String appId,
+ TimelineEntity entity, TimelineWriteResponse response)
+ throws IOException {
PrintWriter out = null;
try {
String dir = mkdirs(outputRoot, ENTITIES_DIR, clusterId, userId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 9f8257f..a384a84 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -35,6 +35,9 @@ import org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderContex
import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader;
import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory;
+/**
+ * HBase based implementation for {@link TimelineReader}.
+ */
public class HBaseTimelineReaderImpl
extends AbstractService implements TimelineReader {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
index 2a82ccb..997b175 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineWriterImpl.java
@@ -93,7 +93,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * initializes the hbase connection to write to the entity table
+ * initializes the hbase connection to write to the entity table.
*/
@Override
protected void serviceInit(Configuration conf) throws Exception {
@@ -104,7 +104,8 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
appToFlowTable = new AppToFlowTable().getTableMutator(hbaseConf, conn);
applicationTable = new ApplicationTable().getTableMutator(hbaseConf, conn);
flowRunTable = new FlowRunTable().getTableMutator(hbaseConf, conn);
- flowActivityTable = new FlowActivityTable().getTableMutator(hbaseConf, conn);
+ flowActivityTable =
+ new FlowActivityTable().getTableMutator(hbaseConf, conn);
}
/**
@@ -289,7 +290,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * Stores the Relations from the {@linkplain TimelineEntity} object
+ * Stores the Relations from the {@linkplain TimelineEntity} object.
*/
private <T> void storeRelations(byte[] rowKey,
Map<String, Set<String>> connectedEntities,
@@ -306,7 +307,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * Stores information from the {@linkplain TimelineEntity} object
+ * Stores information from the {@linkplain TimelineEntity} object.
*/
private void storeInfo(byte[] rowKey, TimelineEntity te, String flowVersion,
boolean isApplication) throws IOException {
@@ -341,7 +342,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * stores the config information from {@linkplain TimelineEntity}
+ * stores the config information from {@linkplain TimelineEntity}.
*/
private void storeConfig(byte[] rowKey, Map<String, String> config,
boolean isApplication) throws IOException {
@@ -351,17 +352,17 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
for (Map.Entry<String, String> entry : config.entrySet()) {
if (isApplication) {
ApplicationColumnPrefix.CONFIG.store(rowKey, applicationTable,
- entry.getKey(), null, entry.getValue());
+ entry.getKey(), null, entry.getValue());
} else {
EntityColumnPrefix.CONFIG.store(rowKey, entityTable, entry.getKey(),
- null, entry.getValue());
+ null, entry.getValue());
}
}
}
/**
* stores the {@linkplain TimelineMetric} information from the
- * {@linkplain TimelineEvent} object
+ * {@linkplain TimelineEvent} object.
*/
private void storeMetrics(byte[] rowKey, Set<TimelineMetric> metrics,
boolean isApplication) throws IOException {
@@ -373,10 +374,10 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
Long timestamp = timeseriesEntry.getKey();
if (isApplication) {
ApplicationColumnPrefix.METRIC.store(rowKey, applicationTable,
- metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
} else {
EntityColumnPrefix.METRIC.store(rowKey, entityTable,
- metricColumnQualifier, timestamp, timeseriesEntry.getValue());
+ metricColumnQualifier, timestamp, timeseriesEntry.getValue());
}
}
}
@@ -384,7 +385,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
}
/**
- * Stores the events from the {@linkplain TimelineEvent} object
+ * Stores the events from the {@linkplain TimelineEvent} object.
*/
private void storeEvents(byte[] rowKey, Set<TimelineEvent> events,
boolean isApplication) throws IOException {
@@ -428,10 +429,10 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
Bytes.toBytes(info.getKey()));
if (isApplication) {
ApplicationColumnPrefix.EVENT.store(rowKey, applicationTable,
- compoundColumnQualifierBytes, null, info.getValue());
+ compoundColumnQualifierBytes, null, info.getValue());
} else {
EntityColumnPrefix.EVENT.store(rowKey, entityTable,
- compoundColumnQualifierBytes, null, info.getValue());
+ compoundColumnQualifierBytes, null, info.getValue());
}
} // for info: eventInfo
}
@@ -459,7 +460,7 @@ public class HBaseTimelineWriterImpl extends AbstractService implements
/**
* close the hbase connections The close APIs perform flushing and release any
- * resources held
+ * resources held.
*/
@Override
protected void serviceStop() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
index e1219e0..1484f22 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/OfflineAggregationWriter.java
@@ -29,7 +29,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.common.OfflineAggre
import java.io.IOException;
/**
- * YARN timeline service v2 offline aggregation storage interface
+ * YARN timeline service v2 offline aggregation storage interface.
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -46,8 +46,8 @@ public abstract class OfflineAggregationWriter extends AbstractService {
/**
* Persist aggregated timeline entities to the offline store based on which
- * track this entity is to be rolled up to. The tracks along which aggregations
- * are to be done are given by {@link OfflineAggregationInfo}.
+ * track this entity is to be rolled up to. The tracks along which
+ * aggregations are to be done are given by {@link OfflineAggregationInfo}.
*
* @param context a {@link TimelineCollectorContext} object that describes the
* context information of the aggregated data. Depends on the
@@ -58,9 +58,10 @@ public abstract class OfflineAggregationWriter extends AbstractService {
* detail of the aggregation. Current supported option is
* {@link OfflineAggregationInfo#FLOW_AGGREGATION}.
* @return a {@link TimelineWriteResponse} object.
- * @throws IOException
+ * @throws IOException if any problem occurs while writing aggregated
+ * entities.
*/
abstract TimelineWriteResponse writeAggregatedEntity(
- TimelineCollectorContext context,
- TimelineEntities entities, OfflineAggregationInfo info) throws IOException;
+ TimelineCollectorContext context, TimelineEntities entities,
+ OfflineAggregationInfo info) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
index b5834c0..130cb6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/PhoenixOfflineAggregationWriterImpl.java
@@ -102,18 +102,18 @@ public class PhoenixOfflineAggregationWriterImpl
private static final String PHOENIX_COL_FAMILY_PLACE_HOLDER
= "timeline_cf_placeholder";
- /** Default Phoenix JDBC driver name */
+ /** Default Phoenix JDBC driver name. */
private static final String DRIVER_CLASS_NAME
= "org.apache.phoenix.jdbc.PhoenixDriver";
- /** Default Phoenix timeline config column family */
+ /** Default Phoenix timeline config column family. */
private static final String METRIC_COLUMN_FAMILY = "m.";
- /** Default Phoenix timeline info column family */
+ /** Default Phoenix timeline info column family. */
private static final String INFO_COLUMN_FAMILY = "i.";
- /** Default separator for Phoenix storage */
+ /** Default separator for Phoenix storage. */
private static final String AGGREGATION_STORAGE_SEPARATOR = ";";
- /** Connection string to the deployed Phoenix cluster */
+ /** Connection string to the deployed Phoenix cluster. */
private String connString = null;
private Properties connProperties = new Properties();
@@ -162,7 +162,8 @@ public class PhoenixOfflineAggregationWriterImpl
}
int idx = info.setStringsForPrimaryKey(ps, context, null, 1);
ps.setLong(idx++, entity.getCreatedTime());
- ps.setString(idx++, StringUtils.join(formattedMetrics.keySet().toArray(),
+ ps.setString(idx++,
+ StringUtils.join(formattedMetrics.keySet().toArray(),
AGGREGATION_STORAGE_SEPARATOR));
ps.execute();
@@ -185,7 +186,7 @@ public class PhoenixOfflineAggregationWriterImpl
* Create Phoenix tables for offline aggregation storage if the tables do not
* exist.
*
- * @throws IOException
+ * @throws IOException if any problem happens while creating Phoenix tables.
*/
public void createPhoenixTables() throws IOException {
// Create tables if necessary
@@ -197,7 +198,8 @@ public class PhoenixOfflineAggregationWriterImpl
+ "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ "flow_name VARCHAR NOT NULL, "
+ "created_time UNSIGNED_LONG, "
- + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER
+ + " VARBINARY, "
+ "metric_names VARCHAR, info_keys VARCHAR "
+ "CONSTRAINT pk PRIMARY KEY("
+ "user, cluster, flow_name))";
@@ -206,7 +208,8 @@ public class PhoenixOfflineAggregationWriterImpl
+ OfflineAggregationInfo.USER_AGGREGATION_TABLE_NAME
+ "(user VARCHAR NOT NULL, cluster VARCHAR NOT NULL, "
+ "created_time UNSIGNED_LONG, "
- + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER + " VARBINARY, "
+ + METRIC_COLUMN_FAMILY + PHOENIX_COL_FAMILY_PLACE_HOLDER
+ + " VARBINARY, "
+ "metric_names VARCHAR, info_keys VARCHAR "
+ "CONSTRAINT pk PRIMARY KEY(user, cluster))";
stmt.executeUpdate(sql);
@@ -251,9 +254,9 @@ public class PhoenixOfflineAggregationWriterImpl
private static class DynamicColumns<K> {
static final String COLUMN_FAMILY_TYPE_BYTES = " VARBINARY";
static final String COLUMN_FAMILY_TYPE_STRING = " VARCHAR";
- String columnFamilyPrefix;
- String type;
- Set<K> columns;
+ private String columnFamilyPrefix;
+ private String type;
+ private Set<K> columns;
public DynamicColumns(String columnFamilyPrefix, String type,
Set<K> keyValues) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
index 955ca80..f0b1e47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineAggregationTrack.java
@@ -20,7 +20,7 @@ package org.apache.hadoop.yarn.server.timelineservice.storage;
/**
* specifies the tracks along which an entity
- * info is to be aggregated on
+ * info is to be aggregated on.
*
*/
public enum TimelineAggregationTrack {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index ccb33b7..e8eabf1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -97,7 +97,8 @@ public interface TimelineReader extends Service {
* <cite>FlowRunEntity</cite>.<br>
* For all other entity types, entity returned is of type
* <cite>TimelineEntity</cite>.
- * @throws IOException
+ * @throws IOException if there is an exception encountered while fetching
+ * entity from backend storage.
*/
TimelineEntity getEntity(TimelineReaderContext context,
TimelineDataToRetrieve dataToRetrieve) throws IOException;
@@ -169,7 +170,8 @@ public interface TimelineReader extends Service {
* <cite>FlowRunEntity</cite>.<br>
* For all other entity types, entities returned are of type
* <cite>TimelineEntity</cite>.
- * @throws IOException
+ * @throws IOException if there is an exception encountered while fetching
+ * entity from backend storage.
*/
Set<TimelineEntity> getEntities(
TimelineReaderContext context,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee803e16/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
index 46bc2e6..33f5449 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineSchemaCreator.java
@@ -53,7 +53,9 @@ import com.google.common.annotations.VisibleForTesting;
*/
@InterfaceAudience.Private
@InterfaceStability.Unstable
-public class TimelineSchemaCreator {
+public final class TimelineSchemaCreator {
+ private TimelineSchemaCreator() {
+ }
final static String NAME = TimelineSchemaCreator.class.getSimpleName();
private static final Log LOG = LogFactory.getLog(TimelineSchemaCreator.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: YARN-4450.
TestTimelineAuthenticationFilter and TestYarnConfigurationFields fail.
Contributed by Sangjin Lee.
Posted by gt...@apache.org.
YARN-4450. TestTimelineAuthenticationFilter and
TestYarnConfigurationFields fail. Contributed by Sangjin Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ceeb2a7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ceeb2a7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ceeb2a7
Branch: refs/heads/YARN-2928
Commit: 9ceeb2a7af267fb6b5c767c746faf50485d006fd
Parents: ef71c1f
Author: Li Lu <gt...@apache.org>
Authored: Mon Dec 14 10:48:39 2015 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:01:41 2016 -0700
----------------------------------------------------------------------
.../yarn/conf/TestYarnConfigurationFields.java | 3 +++
.../yarn/client/api/impl/TimelineClientImpl.java | 18 ++++++++++--------
2 files changed, 13 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ceeb2a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index c92a276..c8e1868 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -94,6 +94,9 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Ignore all YARN Application Timeline Service (version 1) properties
configurationPrefixToSkipCompare.add("yarn.timeline-service.");
+ // skip deprecated RM_SYSTEM_METRICS_PUBLISHER_ENABLED
+ configurationPropsToSkipCompare
+ .add(YarnConfiguration.RM_SYSTEM_METRICS_PUBLISHER_ENABLED);
// Used as Java command line properties, not XML
configurationPrefixToSkipCompare.add("yarn.app.container");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ceeb2a7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index f92e6a7..9979a59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -111,7 +111,6 @@ public class TimelineClientImpl extends TimelineClient {
private ConnectionConfigurator connConfigurator;
private DelegationTokenAuthenticator authenticator;
private DelegationTokenAuthenticatedURL.Token token;
- private URI resURI;
private UserGroupInformation authUgi;
private String doAsUser;
private Configuration configuration;
@@ -552,8 +551,8 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public Long run() throws Exception {
// If the timeline DT to renew is different than cached, replace it.
- // Token to set every time for retry, because when exception happens,
- // DelegationTokenAuthenticatedURL will reset it to null;
+ // Token to set every time for retry, because when exception
+ // happens, DelegationTokenAuthenticatedURL will reset it to null;
if (!timelineDT.equals(token.getDelegationToken())) {
token.setDelegationToken((Token) timelineDT);
}
@@ -562,7 +561,8 @@ public class TimelineClientImpl extends TimelineClient {
connConfigurator);
// If the token service address is not available, fall back to use
// the configured service address.
- final URI serviceURI = isTokenServiceAddrEmpty ? resURI
+ final URI serviceURI = isTokenServiceAddrEmpty ?
+ constructResURI(getConfig(), getTimelineServiceAddress(), false)
: new URI(scheme, null, address.getHostName(),
address.getPort(), RESOURCE_URI_STR_V1, null, null);
return authUrl
@@ -588,9 +588,10 @@ public class TimelineClientImpl extends TimelineClient {
@Override
public Void run() throws Exception {
- // If the timeline DT to cancel is different than cached, replace it.
- // Token to set every time for retry, because when exception happens,
- // DelegationTokenAuthenticatedURL will reset it to null;
+ // If the timeline DT to cancel is different than cached, replace
+ // it.
+ // Token to set every time for retry, because when exception
+ // happens, DelegationTokenAuthenticatedURL will reset it to null;
if (!timelineDT.equals(token.getDelegationToken())) {
token.setDelegationToken((Token) timelineDT);
}
@@ -599,7 +600,8 @@ public class TimelineClientImpl extends TimelineClient {
connConfigurator);
// If the token service address is not available, fall back to use
// the configured service address.
- final URI serviceURI = isTokenServiceAddrEmpty ? resURI
+ final URI serviceURI = isTokenServiceAddrEmpty ?
+ constructResURI(getConfig(), getTimelineServiceAddress(), false)
: new URI(scheme, null, address.getHostName(),
address.getPort(), RESOURCE_URI_STR_V1, null, null);
authUrl.cancelDelegationToken(serviceURI.toURL(), token, doAsUser);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: YARN-4447. Provide a mechanism to
represent complex filters and parse them at the REST layer (Varun Saxena via
sjlee)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/047cde55/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java
new file mode 100644
index 0000000..e991d27
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesUtils.java
@@ -0,0 +1,923 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineCompareOp;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineExistsFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList.Operator;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValueFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineKeyValuesFilter;
+import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
+import org.junit.Test;
+
+import com.google.common.collect.Sets;
+
+public class TestTimelineReaderWebServicesUtils {
+ private static void verifyFilterList(String expr, TimelineFilterList list,
+ TimelineFilterList expectedList) throws Exception {
+ assertNotNull(list);
+ assertTrue("Unexpected List received after parsing expression " + expr +
+ ". Expected=" + expectedList + " but Actual=" + list,
+ list.equals(expectedList));
+ }
+
+ @Test
+ public void testMetricFiltersParsing() throws Exception {
+ String expr = "(((key11 ne 234 AND key12 gt 23) AND " +
+ "(key13 lt 34 OR key14 ge 567)) OR (key21 lt 24 OR key22 le 45))";
+ TimelineFilterList expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "key11", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "key12", 23, true)
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key13", 34, true),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "key14", 567, true)
+ )
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key21", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "key22", 45, true)
+ )
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "abc ene 234";
+ expectedList = new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "abc ne 234";
+ expectedList = new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "abc ne 234 AND def gt 23";
+ expectedList = new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "def", 23, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "(abc ne 234 AND def gt 23)";
+ expectedList = new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "def", 23, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "abc ne 234 AND def gt 23 OR rst lt 24";
+ expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "def", 23, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "rst", 24, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "abc ne 234 AND def gt 23 OR rst lt 24 OR xyz le 456";
+ expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "def", 23, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "rst", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "xyz", 456, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "abc ne 234 AND def gt 23 OR rst lt 24 OR xyz le 456 AND pqr ge 2";
+ expectedList = new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "def", 23, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "rst", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "xyz", 456, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "pqr", 2, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ // Test with unnecessary spaces.
+ expr = " abc ne 234 AND def gt 23 OR rst lt "+
+ " 24 OR xyz le 456 AND pqr ge 2 ";
+ expectedList = new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_THAN,
+ "def", 23, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "rst", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "xyz", 456, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "pqr", 2, true)
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "(((key11 ne 234 AND key12 gt 23 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "key11", 234, false),
+ new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_THAN, "key12", 23, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key13", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "key14", 456, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "key15", 2, true)
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key16", 34, true),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "key17", 567, true)
+ )
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key21", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "key22", 45, true)
+ )
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = " ( ( ( key11 ne 234 AND key12 gt " +
+ "23 OR key13 lt 24 OR key14 le 456 AND key15 ge 2" +
+ " ) AND ( key16 lt 34 OR key17 ge 567 ) ) OR " +
+ "( key21 lt 24 OR key22 le 45 ) ) ";
+ expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineCompareFilter(TimelineCompareOp.NOT_EQUAL,
+ "key11", 234, false),
+ new TimelineCompareFilter(
+ TimelineCompareOp.GREATER_THAN, "key12", 23, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key13", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "key14", 456, true)
+ ),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "key15", 2, true)
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key16", 34, true),
+ new TimelineCompareFilter(TimelineCompareOp.GREATER_OR_EQUAL,
+ "key17", 567, true)
+ )
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineCompareFilter(TimelineCompareOp.LESS_THAN,
+ "key21", 24, true),
+ new TimelineCompareFilter(TimelineCompareOp.LESS_OR_EQUAL,
+ "key22", 45, true)
+ )
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr), expectedList);
+
+ expr = "(((key11 ne 234 AND key12 gt 23 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45)";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Improper brackers. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(((key11 ne 234 AND key12 gt v3 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Non Numeric value. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(((key11 ne (234 AND key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Unexpected opening bracket. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(((k)ey11 ne 234 AND key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Unexpected closing bracket. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(((key11 rs 234 AND key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Improper compare op. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(((key11 ne 234 PI key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Improper op. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(((key11 ne 234 PI key12 gt 3 OR key13 lt 24 OR key14 le 456 " +
+ "AND key15 ge 2) AND (key16 lt 34 OR key17 ge 567)) OR (key21 lt 24 " +
+ "OR key22 le 45))";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Improper op. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(key11 ne 234 AND key12 gt 3)) OR (key13 lt 24 OR key14 le 456)";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Unbalanced brackets. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(key11 rne 234 AND key12 gt 3) OR (key13 lt 24 OR key14 le 456)";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Invalid compareop. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ expr = "(key11 ne 234 AND key12 gt 3) OR (key13 lt 24 OR key14 le";
+ try {
+ TimelineReaderWebServicesUtils.parseMetricFilters(expr);
+ fail("Compareop cant be parsed. Exception should have been thrown.");
+ } catch (TimelineParseException e) {}
+
+ assertNull(TimelineReaderWebServicesUtils.parseMetricFilters(null));
+ assertNull(TimelineReaderWebServicesUtils.parseMetricFilters(" "));
+ }
+
+ @Test
+ public void testConfigFiltersParsing() throws Exception {
+ String expr = "(((key11 ne 234 AND key12 eq val12) AND " +
+ "(key13 ene val13 OR key14 eq 567)) OR (key21 eq val_21 OR key22 eq " +
+ "val.22))";
+ TimelineFilterList expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "key11", "234", false),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key12", "val12", true)
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "key13", "val13", true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key14", "567", true)
+ )
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key21", "val_21", true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key22", "val.22", true)
+ )
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseKVFilters(expr, true), expectedList);
+
+ expr = "abc ne 234 AND def eq 23 OR rst ene 24 OR xyz eq 456 AND pqr eq 2";
+ expectedList = new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", "234", false),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "def", "23", true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "rst", "24", true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "xyz", "456", true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "pqr", "2", true)
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseKVFilters(expr, true), expectedList);
+
+ // Test with unnecessary spaces.
+ expr = " abc ne 234 AND def eq 23 OR rst ene "+
+ " 24 OR xyz eq 456 AND pqr eq 2 ";
+ expectedList = new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", "234", false),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "def", "23", true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "rst", "24", true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "xyz", "456", true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "pqr", "2", true)
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseKVFilters(expr, true), expectedList);
+
+ expr = "abc gt 234 AND def eq 23 OR rst ene 24 OR xyz eq 456 AND pqr eq 2";
+ try {
+ TimelineReaderWebServicesUtils.parseKVFilters(expr, true);
+ fail("Invalid compareop specified for config filters. Should be either" +
+ " eq,ne or ene and exception should have been thrown.");
+ } catch (TimelineParseException e){}
+ }
+ @Test
+ public void testInfoFiltersParsing() throws Exception {
+ String expr = "(((key11 ne 234 AND key12 eq val12) AND " +
+ "(key13 ene val13 OR key14 eq 567)) OR (key21 eq val_21 OR key22 eq " +
+ "5.0))";
+ TimelineFilterList expectedList = new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineFilterList(
+ Operator.AND,
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "key11", 234, false),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key12", "val12", true)
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "key13", "val13", true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key14", 567, true)
+ )
+ ),
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key21", "val_21", true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "key22", 5.0, true)
+ )
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseKVFilters(expr, false), expectedList);
+
+ expr = "abc ne 234 AND def eq 23 OR rst ene 24 OR xyz eq 456 AND pqr eq " +
+ "val.1234";
+ expectedList = new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "def", 23, true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "rst", 24, true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "xyz", 456, true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "pqr", "val.1234", true)
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseKVFilters(expr, false), expectedList);
+
+ // Test with unnecessary spaces.
+ expr = " abc ne 234 AND def eq 23 OR rst ene "+
+ " 24 OR xyz eq 456 AND pqr eq 2 ";
+ expectedList = new TimelineFilterList(
+ new TimelineFilterList(
+ Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "abc", 234, false),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "def", 23, true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.NOT_EQUAL,
+ "rst", 24, true),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "xyz", 456, true)
+ ),
+ new TimelineKeyValueFilter(TimelineCompareOp.EQUAL,
+ "pqr", 2, true)
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseKVFilters(expr, false), expectedList);
+ }
+
+ @Test
+ public void testEventFiltersParsing() throws Exception {
+ String expr = "abc,def";
+ TimelineFilterList expectedList = new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "abc"),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "def")
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseEventFilters(expr), expectedList);
+
+ expr = "(abc,def)";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseEventFilters(expr), expectedList);
+
+ expr = "(abc,def) OR (rst, uvx)";
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "abc"),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "def")
+ ),
+ new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "rst"),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "uvx")
+ )
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseEventFilters(expr), expectedList);
+
+ expr = "!(abc,def,uvc) OR (rst, uvx)";
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.NOT_EQUAL, "abc"),
+ new TimelineExistsFilter(TimelineCompareOp.NOT_EQUAL, "def"),
+ new TimelineExistsFilter(TimelineCompareOp.NOT_EQUAL, "uvc")
+ ),
+ new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "rst"),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "uvx")
+ )
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseEventFilters(expr), expectedList);
+
+ expr = "(((!(abc,def,uvc) OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
+ " OR ((bcd,tyu) AND uvb))";
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "abc"),
+ new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "def"),
+ new TimelineExistsFilter(TimelineCompareOp.NOT_EQUAL, "uvc")
+ ),
+ new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "rst"),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "uvx")
+ )
+ ),
+ new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "abcdefg")
+ ),
+ new TimelineFilterList(
+ new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "ghj"),
+ new TimelineExistsFilter(
+ TimelineCompareOp.NOT_EQUAL, "tyu")
+ )
+ )
+ ),
+ new TimelineFilterList(
+ new TimelineFilterList(
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "bcd"),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "tyu")
+ ),
+ new TimelineExistsFilter(TimelineCompareOp.EQUAL, "uvb")
+ )
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseEventFilters(expr), expectedList);
+
+ expr = " ( ( ( ! ( abc , def , uvc) OR ( rst , uvx ) )" +
+ " AND ( ! ( abcdefg ) OR ! ( ghj, tyu) ) ) OR ( (" +
+ " bcd , tyu ) AND uvb ) )";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseEventFilters(expr), expectedList);
+
+ expr = "(((!(abc,def,uvc) OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
+ " OR ((bcd,tyu) AND uvb)";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Improper brackets. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "(((!(abc,def,uvc) (OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
+ " OR ((bcd,tyu) AND uvb))";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected opening bracket. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "(((!(abc,def,uvc) OR) (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
+ " OR ((bcd,tyu) AND uvb))";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected closing bracket. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "(((!(abc,def,uvc) PI (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
+ " OR ((bcd,tyu) AND uvb))";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Invalid op. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "(((!(abc,def,uvc) !OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu)))" +
+ " OR ((bcd,tyu) AND uvb))";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected ! char. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "abc,def,uvc) OR (rst, uvx)";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected closing bracket. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "abc,def,uvc OR )rst, uvx)";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected closing bracket. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "abc,def,uvc OR ,rst, uvx)";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected delimiter. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "abc,def,uvc OR ! ";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unexpected not char. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "(abc,def,uvc)) OR (rst, uvx)";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("Unbalanced brackets. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "(((! ,(abc,def,uvc) OR (rst, uvx)) AND (!(abcdefg) OR !(ghj,tyu" +
+ "))) OR ((bcd,tyu) AND uvb))";
+ try {
+ TimelineReaderWebServicesUtils.parseEventFilters(expr);
+ fail("( should follow ! char. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ assertNull(TimelineReaderWebServicesUtils.parseEventFilters(null));
+ assertNull(TimelineReaderWebServicesUtils.parseEventFilters(" "));
+ }
+
+ @Test
+ public void testRelationFiltersParsing() throws Exception {
+ String expr = "type1:entity11,type2:entity21:entity22";
+ TimelineFilterList expectedList = new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type1", Sets.newHashSet((Object)"entity11")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type2", Sets.newHashSet((Object)"entity21", "entity22"))
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseRelationFilters(expr), expectedList);
+
+ expr = "(type1:entity11,type2:entity21:entity22)";
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseRelationFilters(expr), expectedList);
+
+ expr = "(type1:entity11,type2:entity21:entity22) OR (type3:entity31:" +
+ "entity32:entity33,type1:entity11:entity12)";
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type1", Sets.newHashSet((Object)"entity11")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type2", Sets.newHashSet((Object)"entity21", "entity22"))
+ ),
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type3", Sets.newHashSet(
+ (Object)"entity31", "entity32", "entity33")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type1", Sets.newHashSet((Object)"entity11", "entity12"))
+ )
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseRelationFilters(expr), expectedList);
+
+ expr = "!(type1:entity11,type2:entity21:entity22,type5:entity51) OR " +
+ "(type3:entity31:entity32:entity33,type1:entity11:entity12)";
+
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type1", Sets.newHashSet((Object)"entity11")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type2", Sets.newHashSet((Object)"entity21", "entity22")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type5", Sets.newHashSet((Object)"entity51"))
+ ),
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type3", Sets.newHashSet(
+ (Object)"entity31", "entity32", "entity33")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type1", Sets.newHashSet((Object)"entity11", "entity12"))
+ )
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseRelationFilters(expr), expectedList);
+
+ expr = "(((!(type1:entity11,type2:entity21:entity22,type5:entity51) OR " +
+ "(type3:entity31:entity32:entity33,type1:entity11:entity12)) AND "+
+ "(!(type11:entity111) OR !(type4:entity43:entity44:entity47:entity49," +
+ "type7:entity71))) OR ((type2:entity2,type8:entity88) AND t9:e:e1))";
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type1", Sets.newHashSet((Object)"entity11")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type2", Sets.newHashSet(
+ (Object)"entity21", "entity22")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type5", Sets.newHashSet((Object)"entity51"))
+ ),
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type3", Sets.newHashSet(
+ (Object)"entity31", "entity32", "entity33")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type1", Sets.newHashSet(
+ (Object)"entity11", "entity12"))
+ )
+ ),
+ new TimelineFilterList(Operator.OR,
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type11", Sets.newHashSet((Object)"entity111"))
+ ),
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type4", Sets.newHashSet((Object)"entity43","entity44",
+ "entity47","entity49")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.NOT_EQUAL,
+ "type7", Sets.newHashSet((Object)"entity71"))
+ )
+ )
+ ),
+ new TimelineFilterList(
+ new TimelineFilterList(
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type2", Sets.newHashSet((Object)"entity2")),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL,
+ "type8", Sets.newHashSet((Object)"entity88"))
+ ),
+ new TimelineKeyValuesFilter(TimelineCompareOp.EQUAL, "t9",
+ Sets.newHashSet((Object)"e", "e1"))
+ )
+ );
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseRelationFilters(expr), expectedList);
+
+ expr = " ( ( ( ! ( type1:entity11 , type2:entity21:entity22" +
+ " , type5:entity51 ) OR ( type3:entity31:entity32:entity33 " +
+ " , type1:entity11:entity12)) AND (!( type11:entity111 ) OR " +
+ " ! ( type4:entity43:entity44:entity47:entity49 , " +
+ "type7:entity71 ) ) ) OR ( ( type2:entity2 , type8:entity88) " +
+ "AND t9:e:e1 ) ) ";
+ verifyFilterList(expr, TimelineReaderWebServicesUtils.
+ parseRelationFilters(expr), expectedList);
+
+ expr = "(((!(type1 : entity11,type2:entity21:entity22,type5:entity51) OR " +
+ "(type3:entity31:entity32:entity33,type1:entity11:entity12)) AND "+
+ "(!(type11:entity111) OR !(type4:entity43:entity44:entity47:entity49," +
+ "type7:entity71))) OR ((type2:entity2,type8:entity88) AND t9:e:e1))";
+ try {
+ TimelineReaderWebServicesUtils.parseRelationFilters(expr);
+ fail("Space not allowed in relation expression. Exception should have " +
+ "been thrown");
+ } catch (TimelineParseException e){}
+ }
+
+ @Test
+ public void testDataToRetrieve() throws Exception {
+ String expr = "abc,def";
+ TimelineFilterList expectedList = new TimelineFilterList(Operator.OR,
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "abc"),
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "def")
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "(abc,def)";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = " ( abc , def ) ";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = " abc , def ";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "!(abc,def)";
+ expectedList = new TimelineFilterList(
+ new TimelinePrefixFilter(TimelineCompareOp.NOT_EQUAL, "abc"),
+ new TimelinePrefixFilter(TimelineCompareOp.NOT_EQUAL, "def")
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = " ! ( abc , def ) ";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "!(abc)";
+ expectedList = new TimelineFilterList(
+ new TimelinePrefixFilter(TimelineCompareOp.NOT_EQUAL, "abc")
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "(abc)";
+ expectedList = new TimelineFilterList(Operator.OR,
+ new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "abc")
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "abc";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = " ! ( abc , def , xyz) ";
+ expectedList = new TimelineFilterList(
+ new TimelinePrefixFilter(TimelineCompareOp.NOT_EQUAL, "abc"),
+ new TimelinePrefixFilter(TimelineCompareOp.NOT_EQUAL, "def"),
+ new TimelinePrefixFilter(TimelineCompareOp.NOT_EQUAL, "xyz")
+ );
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "!(abc,def,xyz)";
+ verifyFilterList(expr,
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr), expectedList);
+
+ expr = "!(abc,def,xyz";
+ try {
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
+ fail("No closing bracket. Exception should have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "!abc,def,xyz";
+ try {
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
+ fail("NOT(!) should be followed by opening bracket. Exception should " +
+ "have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "!abc,def,xyz";
+ try {
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
+ fail("NOT(!) should be followed by opening bracket. Exception should " +
+ "have been thrown");
+ } catch (TimelineParseException e){}
+
+ expr = "! r( abc,def,xyz)";
+ try {
+ TimelineReaderWebServicesUtils.parseDataToRetrieve(expr);
+ fail("NOT(!) should be followed by opening bracket. Exception should " +
+ "have been thrown");
+ } catch (TimelineParseException e){}
+
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve(null));
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve(" "));
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve("()"));
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve("!()"));
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve("( )"));
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve("!( )"));
+ assertNull(TimelineReaderWebServicesUtils.parseDataToRetrieve("! ( )"));
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: YARN-3461. Consolidate flow
name/version/run defaults. (Sangjin Lee via Varun Saxena)
Posted by gt...@apache.org.
YARN-3461. Consolidate flow name/version/run defaults. (Sangjin Lee via Varun Saxena)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/408f0014
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/408f0014
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/408f0014
Branch: refs/heads/YARN-2928
Commit: 408f00144b8e6808eed91c5549e60e63c50d052d
Parents: 899468d
Author: Varun Saxena <va...@apache.org>
Authored: Thu Apr 7 22:10:11 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:28 2016 -0700
----------------------------------------------------------------------
.../mapred/TestMRTimelineEventHandling.java | 46 +++++++-------
.../distributedshell/TestDistributedShell.java | 18 ++++--
.../yarn/util/timeline/TimelineUtils.java | 8 ++-
.../resourcemanager/amlauncher/AMLauncher.java | 67 +++++++++++++++-----
.../RMTimelineCollectorManager.java | 36 +++++++++--
.../TestSystemMetricsPublisherForV2.java | 20 +++---
.../collector/AppLevelTimelineCollector.java | 11 +---
.../collector/NodeTimelineCollectorManager.java | 12 ++++
.../collector/TimelineCollectorContext.java | 5 +-
9 files changed, 148 insertions(+), 75 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index f7283ae..300b4fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -20,15 +20,12 @@ package org.apache.hadoop.mapred;
import java.io.File;
import java.io.IOException;
-
import java.util.EnumSet;
import java.util.List;
-import java.util.Set;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapreduce.MRJobConfig;
@@ -38,9 +35,9 @@ import org.apache.hadoop.mapreduce.v2.MiniMRYarnCluster;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
@@ -48,7 +45,6 @@ import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timelineservice.collector.PerNodeTimelineCollectorsAuxService;
import org.apache.hadoop.yarn.server.timelineservice.storage.FileSystemTimelineWriterImpl;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-
import org.junit.Assert;
import org.junit.Test;
@@ -205,7 +201,7 @@ public class TestMRTimelineEventHandling {
ApplicationReport appReport = apps.get(0);
firstAppId = appReport.getApplicationId();
- checkNewTimelineEvent(firstAppId);
+ checkNewTimelineEvent(firstAppId, appReport);
LOG.info("Run 2nd job which should be failed.");
job = UtilsForTests.runJobFail(new JobConf(conf), inDir, outDir);
@@ -214,11 +210,10 @@ public class TestMRTimelineEventHandling {
apps = yarnClient.getApplications(appStates);
Assert.assertEquals(apps.size(), 2);
-
- ApplicationId secAppId = null;
- secAppId = apps.get(0).getApplicationId() == firstAppId ?
- apps.get(1).getApplicationId() : apps.get(0).getApplicationId();
- checkNewTimelineEvent(firstAppId);
+
+ appReport = apps.get(0).getApplicationId().equals(firstAppId) ?
+ apps.get(0) : apps.get(1);
+ checkNewTimelineEvent(firstAppId, appReport);
} finally {
if (cluster != null) {
@@ -235,7 +230,8 @@ public class TestMRTimelineEventHandling {
}
}
- private void checkNewTimelineEvent(ApplicationId appId) throws IOException {
+ private void checkNewTimelineEvent(ApplicationId appId,
+ ApplicationReport appReport) throws IOException {
String tmpRoot =
FileSystemTimelineWriterImpl.DEFAULT_TIMELINE_SERVICE_STORAGE_DIR_ROOT
+ "/entities/";
@@ -243,15 +239,18 @@ public class TestMRTimelineEventHandling {
File tmpRootFolder = new File(tmpRoot);
Assert.assertTrue(tmpRootFolder.isDirectory());
- String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
- UserGroupInformation.getCurrentUser().getShortUserName() +
- "/" + TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId) +
- "/1/1/" + appId.toString();
+ String basePath = tmpRoot + YarnConfiguration.DEFAULT_RM_CLUSTER_ID +
+ "/" + UserGroupInformation.getCurrentUser().getShortUserName() +
+ "/" + appReport.getName() +
+ "/" + TimelineUtils.DEFAULT_FLOW_VERSION +
+ "/" + appReport.getStartTime() +
+ "/" + appId.toString();
// for this test, we expect MAPREDUCE_JOB and MAPREDUCE_TASK dirs
String outputDirJob = basePath + "/MAPREDUCE_JOB/";
File entityFolder = new File(outputDirJob);
- Assert.assertTrue("Job output directory: " + outputDirJob + " is not exist.",
+ Assert.assertTrue("Job output directory: " + outputDirJob +
+ " does not exist.",
entityFolder.isDirectory());
// check for job event file
@@ -260,13 +259,15 @@ public class TestMRTimelineEventHandling {
String jobEventFilePath = outputDirJob + jobEventFileName;
File jobEventFile = new File(jobEventFilePath);
- Assert.assertTrue("jobEventFilePath: " + jobEventFilePath + " is not exist.",
+ Assert.assertTrue("jobEventFilePath: " + jobEventFilePath +
+ " does not exist.",
jobEventFile.exists());
// check for task event file
String outputDirTask = basePath + "/MAPREDUCE_TASK/";
File taskFolder = new File(outputDirTask);
- Assert.assertTrue("Task output directory: " + outputDirTask + " is not exist.",
+ Assert.assertTrue("Task output directory: " + outputDirTask +
+ " does not exist.",
taskFolder.isDirectory());
String taskEventFileName = appId.toString().replaceAll("application", "task")
@@ -274,14 +275,15 @@ public class TestMRTimelineEventHandling {
String taskEventFilePath = outputDirTask + taskEventFileName;
File taskEventFile = new File(taskEventFilePath);
- Assert.assertTrue("taskEventFileName: " + taskEventFilePath + " is not exist.",
+ Assert.assertTrue("taskEventFileName: " + taskEventFilePath +
+ " does not exist.",
taskEventFile.exists());
// check for task attempt event file
String outputDirTaskAttempt = basePath + "/MAPREDUCE_TASK_ATTEMPT/";
File taskAttemptFolder = new File(outputDirTaskAttempt);
Assert.assertTrue("TaskAttempt output directory: " + outputDirTaskAttempt +
- " is not exist.", taskAttemptFolder.isDirectory());
+ " does not exist.", taskAttemptFolder.isDirectory());
String taskAttemptEventFileName = appId.toString().replaceAll(
"application", "attempt") + "_m_000000_0" +
@@ -291,7 +293,7 @@ public class TestMRTimelineEventHandling {
taskAttemptEventFileName;
File taskAttemptEventFile = new File(taskAttemptEventFilePath);
Assert.assertTrue("taskAttemptEventFileName: " + taskAttemptEventFilePath +
- " is not exist.", taskAttemptEventFile.exists());
+ " does not exist.", taskAttemptEventFile.exists());
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 203a7fc..4a401e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -372,13 +372,14 @@ public class TestDistributedShell {
boolean verified = false;
String errorMessage = "";
ApplicationId appId = null;
+ ApplicationReport appReport = null;
while(!verified) {
List<ApplicationReport> apps = yarnClient.getApplications();
if (apps.size() == 0 ) {
Thread.sleep(10);
continue;
}
- ApplicationReport appReport = apps.get(0);
+ appReport = apps.get(0);
appId = appReport.getApplicationId();
if(appReport.getHost().equals("N/A")) {
Thread.sleep(10);
@@ -424,7 +425,7 @@ public class TestDistributedShell {
if (!isTestingTimelineV2) {
checkTimelineV1(haveDomain);
} else {
- checkTimelineV2(haveDomain, appId, defaultFlow);
+ checkTimelineV2(haveDomain, appId, defaultFlow, appReport);
}
}
@@ -481,7 +482,7 @@ public class TestDistributedShell {
}
private void checkTimelineV2(boolean haveDomain, ApplicationId appId,
- boolean defaultFlow) throws Exception {
+ boolean defaultFlow, ApplicationReport appReport) throws Exception {
LOG.info("Started checkTimelineV2 ");
// For PoC check in /tmp/timeline_service_data YARN-3264
String tmpRoot =
@@ -494,10 +495,13 @@ public class TestDistributedShell {
String basePath = tmpRoot +
YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/" +
UserGroupInformation.getCurrentUser().getShortUserName() +
- (defaultFlow ? "/" +
- TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId) +
- "/1/1/" : "/test_flow_name/test_flow_version/12345678/") +
- appId.toString();
+ (defaultFlow ?
+ "/" + appReport.getName() + "/" +
+ TimelineUtils.DEFAULT_FLOW_VERSION +"/" +
+ appReport.getStartTime() +"/" :
+ "/test_flow_name/test_flow_version/12345678/") +
+ appId.toString();
+ LOG.info("basePath: " + basePath);
// for this test, we expect DS_APP_ATTEMPT AND DS_CONTAINER dirs
// Verify DS_APP_ATTEMPT entities posted by the client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
index 6faeffa..b618ac1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
@@ -49,6 +49,7 @@ public class TimelineUtils {
"TIMELINE_FLOW_VERSION_TAG";
public static final String FLOW_RUN_ID_TAG_PREFIX =
"TIMELINE_FLOW_RUN_ID_TAG";
+ public final static String DEFAULT_FLOW_VERSION = "1";
private static ObjectMapper mapper;
@@ -162,9 +163,12 @@ public class TimelineUtils {
return SecurityUtil.buildTokenService(timelineServiceAddr);
}
- public static String generateDefaultFlowNameBasedOnAppId(
+ public static String generateDefaultFlowName(String appName,
ApplicationId appId) {
- return "flow_" + appId.getClusterTimestamp() + "_" + appId.getId();
+ return (appName != null &&
+ !appName.equals(YarnConfiguration.DEFAULT_APPLICATION_NAME)) ?
+ appName :
+ "flow_" + appId.getClusterTimestamp() + "_" + appId.getId();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 9a7638c..f5ed662 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -195,6 +196,8 @@ public class AMLauncher implements Runnable {
// Finalize the container
setupTokens(container, containerID);
+ // set the flow context optionally for timeline service v.2
+ setFlowContext(container);
return container;
}
@@ -216,15 +219,6 @@ public class AMLauncher implements Runnable {
.get(applicationId)
.getSubmitTime()));
- if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
- // Set flow context info
- for (String tag :
- rmContext.getRMApps().get(applicationId).getApplicationTags()) {
- setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX, tag);
- setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX, tag);
- setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX, tag);
- }
- }
Credentials credentials = new Credentials();
DataInputByteBuffer dibb = new DataInputByteBuffer();
ByteBuffer tokens = container.getTokens();
@@ -245,17 +239,58 @@ public class AMLauncher implements Runnable {
container.setTokens(ByteBuffer.wrap(dob.getData(), 0, dob.getLength()));
}
- private static void setFlowTags(
- Map<String, String> environment, String tagPrefix, String tag) {
- if (tag.startsWith(tagPrefix + ":") ||
- tag.startsWith(tagPrefix.toLowerCase() + ":")) {
- String value = tag.substring(tagPrefix.length() + 1);
- if (!value.isEmpty()) {
- environment.put(tagPrefix, value);
+ private void setFlowContext(ContainerLaunchContext container) {
+ if (YarnConfiguration.timelineServiceV2Enabled(conf)) {
+ Map<String, String> environment = container.getEnvironment();
+ ApplicationId applicationId =
+ application.getAppAttemptId().getApplicationId();
+ RMApp app = rmContext.getRMApps().get(applicationId);
+
+ // initialize the flow in the environment with default values for those
+ // that do not specify the flow tags
+ // flow name: app name (or app id if app name is missing),
+ // flow version: "1", flow run id: start time
+ setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX,
+ TimelineUtils.generateDefaultFlowName(app.getName(), applicationId));
+ setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX,
+ TimelineUtils.DEFAULT_FLOW_VERSION);
+ setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX,
+ String.valueOf(app.getStartTime()));
+
+ // Set flow context info: the flow context is received via the application
+ // tags
+ for (String tag : app.getApplicationTags()) {
+ String[] parts = tag.split(":", 2);
+ if (parts.length != 2 || parts[1].isEmpty()) {
+ continue;
+ }
+ switch (parts[0].toUpperCase()) {
+ case TimelineUtils.FLOW_NAME_TAG_PREFIX:
+ setFlowTags(environment, TimelineUtils.FLOW_NAME_TAG_PREFIX,
+ parts[1]);
+ break;
+ case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
+ setFlowTags(environment, TimelineUtils.FLOW_VERSION_TAG_PREFIX,
+ parts[1]);
+ break;
+ case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
+ setFlowTags(environment, TimelineUtils.FLOW_RUN_ID_TAG_PREFIX,
+ parts[1]);
+ break;
+ default:
+ break;
+ }
}
}
}
+ private static void setFlowTags(
+ Map<String, String> environment, String tagPrefix, String value) {
+ if (!value.isEmpty()) {
+ environment.put(tagPrefix, value);
+ }
+ }
+
@VisibleForTesting
protected Token<AMRMTokenIdentifier> createAndSetAMRMToken() {
Token<AMRMTokenIdentifier> amrmToken =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
index ff055a1..a4f1084 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/timelineservice/RMTimelineCollectorManager.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.resourcemanager.timelineservice;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -25,6 +27,7 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollector;
+import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorContext;
import org.apache.hadoop.yarn.server.timelineservice.collector.TimelineCollectorManager;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -35,6 +38,9 @@ import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class RMTimelineCollectorManager extends TimelineCollectorManager {
+ private static final Log LOG =
+ LogFactory.getLog(RMTimelineCollectorManager.class);
+
private RMContext rmContext;
public RMTimelineCollectorManager(RMContext rmContext) {
@@ -51,9 +57,21 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
"non-existing app " + appId);
}
String userId = app.getUser();
+ TimelineCollectorContext context = collector.getTimelineEntityContext();
if (userId != null && !userId.isEmpty()) {
- collector.getTimelineEntityContext().setUserId(userId);
+ context.setUserId(userId);
}
+
+ // initialize the flow in the environment with default values for those
+ // that do not specify the flow tags
+ // flow name: app name (or app id if app name is missing),
+ // flow version: "1", flow run id: start time
+ context.setFlowName(TimelineUtils.generateDefaultFlowName(
+ app.getName(), appId));
+ context.setFlowVersion(TimelineUtils.DEFAULT_FLOW_VERSION);
+ context.setFlowRunId(app.getStartTime());
+
+ // the flow context is received via the application tags
for (String tag : app.getApplicationTags()) {
String[] parts = tag.split(":", 2);
if (parts.length != 2 || parts[1].isEmpty()) {
@@ -61,14 +79,22 @@ public class RMTimelineCollectorManager extends TimelineCollectorManager {
}
switch (parts[0].toUpperCase()) {
case TimelineUtils.FLOW_NAME_TAG_PREFIX:
- collector.getTimelineEntityContext().setFlowName(parts[1]);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the flow name: " + parts[1]);
+ }
+ context.setFlowName(parts[1]);
break;
case TimelineUtils.FLOW_VERSION_TAG_PREFIX:
- collector.getTimelineEntityContext().setFlowVersion(parts[1]);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the flow version: " + parts[1]);
+ }
+ context.setFlowVersion(parts[1]);
break;
case TimelineUtils.FLOW_RUN_ID_TAG_PREFIX:
- collector.getTimelineEntityContext().setFlowRunId(
- Long.parseLong(parts[1]));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the flow run id: " + parts[1]);
+ }
+ context.setFlowRunId(Long.parseLong(parts[1]));
break;
default:
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
index 1a58cb2..7a018ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisherForV2.java
@@ -83,8 +83,6 @@ public class TestSystemMetricsPublisherForV2 {
private static TimelineServiceV2Publisher metricsPublisher;
private static DrainDispatcher dispatcher = new DrainDispatcher();
- private static final String DEFAULT_FLOW_VERSION = "1";
- private static final long DEFAULT_FLOW_RUN = 1;
private static ConcurrentMap<ApplicationId, RMApp> rmAppsMapInContext;
@@ -316,16 +314,14 @@ public class TestSystemMetricsPublisherForV2 {
private String getTimelineEntityDir(RMApp app) {
String outputDirApp =
- testRootDir.getAbsolutePath()+"/"
- + FileSystemTimelineWriterImpl.ENTITIES_DIR
- + "/"
- + YarnConfiguration.DEFAULT_RM_CLUSTER_ID
- + "/"
- + app.getUser()
- + "/"
- + TimelineUtils.generateDefaultFlowNameBasedOnAppId(app
- .getApplicationId()) + "/" + DEFAULT_FLOW_VERSION + "/"
- + DEFAULT_FLOW_RUN + "/" + app.getApplicationId();
+ testRootDir.getAbsolutePath() + "/"
+ + FileSystemTimelineWriterImpl.ENTITIES_DIR + "/"
+ + YarnConfiguration.DEFAULT_RM_CLUSTER_ID + "/"
+ + app.getUser() + "/"
+ + app.getName() + "/"
+ + TimelineUtils.DEFAULT_FLOW_VERSION + "/"
+ + app.getStartTime() + "/"
+ + app.getApplicationId();
return outputDirApp;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
index 36dd7b0..4fe445a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/AppLevelTimelineCollector.java
@@ -18,14 +18,14 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
-import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
+
+import com.google.common.base.Preconditions;
/**
* Service that handles writes to the timeline service and writes them to the
@@ -54,13 +54,6 @@ public class AppLevelTimelineCollector extends TimelineCollector {
// context info from NM.
// Current user usually is not the app user, but keep this field non-null
context.setUserId(UserGroupInformation.getCurrentUser().getShortUserName());
- // Use app ID to generate a default flow name for orphan app
- context.setFlowName(
- TimelineUtils.generateDefaultFlowNameBasedOnAppId(appId));
- // Set the flow version to string 1 if it's an orphan app
- context.setFlowVersion("1");
- // Set the flow run ID to 1 if it's an orphan app
- context.setFlowRunId(1L);
context.setAppId(appId.toString());
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
index 785fb19..75557a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/NodeTimelineCollectorManager.java
@@ -164,18 +164,30 @@ public class NodeTimelineCollectorManager extends TimelineCollectorManager {
getNMCollectorService().getTimelineCollectorContext(request);
String userId = response.getUserId();
if (userId != null && !userId.isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the user in the context: " + userId);
+ }
collector.getTimelineEntityContext().setUserId(userId);
}
String flowName = response.getFlowName();
if (flowName != null && !flowName.isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the flow name: " + flowName);
+ }
collector.getTimelineEntityContext().setFlowName(flowName);
}
String flowVersion = response.getFlowVersion();
if (flowVersion != null && !flowVersion.isEmpty()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the flow version: " + flowVersion);
+ }
collector.getTimelineEntityContext().setFlowVersion(flowVersion);
}
long flowRunId = response.getFlowRunId();
if (flowRunId != 0L) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Setting the flow run id: " + flowRunId);
+ }
collector.getTimelineEntityContext().setFlowRunId(flowRunId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/408f0014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
index 58d68df..981ee2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/collector/TimelineCollectorContext.java
@@ -19,12 +19,12 @@
package org.apache.hadoop.yarn.server.timelineservice.collector;
import org.apache.hadoop.yarn.server.timelineservice.TimelineContext;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
/**
* Encapsulates context information required by collector during a put.
*/
public class TimelineCollectorContext extends TimelineContext {
-
private String flowVersion;
public TimelineCollectorContext() {
@@ -34,7 +34,8 @@ public class TimelineCollectorContext extends TimelineContext {
public TimelineCollectorContext(String clusterId, String userId,
String flowName, String flowVersion, Long flowRunId, String appId) {
super(clusterId, userId, flowName, flowRunId, appId);
- this.flowVersion = flowVersion;
+ this.flowVersion = flowVersion == null ?
+ TimelineUtils.DEFAULT_FLOW_VERSION : flowVersion;
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-4644. TestRMRestart fails and
findbugs issue in YARN-2928 branch (Varun Saxena via Naganarasimha G R)
Posted by gt...@apache.org.
YARN-4644. TestRMRestart fails and findbugs issue in YARN-2928 branch (Varun Saxena via Naganarasimha G R)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e02094f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e02094f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e02094f2
Branch: refs/heads/YARN-2928
Commit: e02094f28526452387dea6e1ec2500bfef5696fc
Parents: 97d5cf3
Author: Naganarasimha <na...@apache.com>
Authored: Thu Jan 28 17:54:15 2016 +0530
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:24:17 2016 -0700
----------------------------------------------------------------------
.../yarn/server/resourcemanager/ResourceTrackerService.java | 2 --
.../hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java | 3 ---
2 files changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e02094f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index fea8183..459e297 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -531,8 +531,6 @@ public class ResourceTrackerService extends AbstractService implements
}
// 4. Send status to RMNode, saving the latest response.
- List<ApplicationId> keepAliveApps =
- remoteNodeStatus.getKeepAliveApplications();
RMNodeStatusEvent nodeStatusEvent =
new RMNodeStatusEvent(nodeId, remoteNodeStatus, nodeHeartBeatResponse);
if (request.getLogAggregationReportsForApps() != null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e02094f2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index a7df5a4..cea7563 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -891,9 +891,6 @@ public class RMAppImpl implements RMApp, Recoverable {
this.firstAttemptIdInStateStore = appState.getFirstAttemptId();
this.nextAttemptId = firstAttemptIdInStateStore;
}
-
- // send the ATS create Event
- sendATSCreateEvent(this, this.startTime);
//TODO recover collector address.
//this.collectorAddr = appState.getCollectorAddr();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: YARN-4224. Support fetching
entities by UID and change the REST interface to conform to current REST
APIs' in YARN. (Varun Saxena via gtcarrera9)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
index a054ee5..7a70de8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServices.java
@@ -18,17 +18,12 @@
package org.apache.hadoop.yarn.server.timelineservice.reader;
-import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Collections;
import java.util.Date;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Locale;
-import java.util.Map;
import java.util.Set;
import java.util.TimeZone;
@@ -51,10 +46,11 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Time;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineAbout;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
@@ -63,7 +59,7 @@ import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.common.annotations.VisibleForTesting;
import com.google.inject.Singleton;
-/** REST end point for Timeline Reader */
+/** REST end point for Timeline Reader. */
@Private
@Unstable
@Singleton
@@ -167,117 +163,6 @@ public class TimelineReaderWebServices {
}
}
- private static Set<String> parseValuesStr(String str, String delimiter) {
- if (str == null || str.isEmpty()) {
- return null;
- }
- Set<String> strSet = new HashSet<String>();
- String[] strs = str.split(delimiter);
- for (String aStr : strs) {
- strSet.add(aStr.trim());
- }
- return strSet;
- }
-
- @SuppressWarnings("unchecked")
- private static <T> void parseKeyValues(Map<String,T> map, String str,
- String pairsDelim, String keyValuesDelim, boolean stringValue,
- boolean multipleValues) {
- String[] pairs = str.split(pairsDelim);
- for (String pair : pairs) {
- if (pair == null || pair.trim().isEmpty()) {
- continue;
- }
- String[] pairStrs = pair.split(keyValuesDelim);
- if (pairStrs.length < 2) {
- continue;
- }
- if (!stringValue) {
- try {
- Object value =
- GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
- map.put(pairStrs[0].trim(), (T) value);
- } catch (IOException e) {
- map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
- }
- } else {
- String key = pairStrs[0].trim();
- if (multipleValues) {
- Set<String> values = new HashSet<String>();
- for (int i = 1; i < pairStrs.length; i++) {
- values.add(pairStrs[i].trim());
- }
- map.put(key, (T) values);
- } else {
- map.put(key, (T) pairStrs[1].trim());
- }
- }
- }
- }
-
- private static Map<String, Set<String>> parseKeyStrValuesStr(String str,
- String pairsDelim, String keyValuesDelim) {
- if (str == null) {
- return null;
- }
- Map<String, Set<String>> map = new HashMap<String, Set<String>>();
- parseKeyValues(map, str,pairsDelim, keyValuesDelim, true, true);
- return map;
- }
-
- private static Map<String, String> parseKeyStrValueStr(String str,
- String pairsDelim, String keyValDelim) {
- if (str == null) {
- return null;
- }
- Map<String, String> map = new HashMap<String, String>();
- parseKeyValues(map, str, pairsDelim, keyValDelim, true, false);
- return map;
- }
-
- private static Map<String, Object> parseKeyStrValueObj(String str,
- String pairsDelim, String keyValDelim) {
- if (str == null) {
- return null;
- }
- Map<String, Object> map = new HashMap<String, Object>();
- parseKeyValues(map, str, pairsDelim, keyValDelim, false, false);
- return map;
- }
-
- private static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
- if (str == null) {
- return null;
- }
- String[] strs = str.split(delimiter);
- EnumSet<Field> fieldList = EnumSet.noneOf(Field.class);
- for (String s : strs) {
- fieldList.add(Field.valueOf(s.trim().toUpperCase()));
- }
- return fieldList;
- }
-
- private static Long parseLongStr(String str) {
- return str == null ? null : Long.parseLong(str.trim());
- }
-
- private static String parseStr(String str) {
- return str == null ? null : str.trim();
- }
-
- private static UserGroupInformation getUser(HttpServletRequest req) {
- String remoteUser = req.getRemoteUser();
- UserGroupInformation callerUGI = null;
- if (remoteUser != null) {
- callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
- }
- return callerUGI;
- }
-
- private static String getUserName(UserGroupInformation callerUGI) {
- return ((callerUGI != null) ? callerUGI.getUserName().trim() : "");
- }
-
private TimelineReaderManager getTimelineReaderManager() {
return (TimelineReaderManager)
ctxt.getAttribute(TimelineReaderServer.TIMELINE_READER_MANAGER_ATTR);
@@ -294,6 +179,10 @@ public class TimelineReaderWebServices {
} else if (e instanceof IllegalArgumentException) {
throw new BadRequestException(e.getMessage() == null ?
"Requested Invalid Field." : e.getMessage());
+ } else if (e instanceof NotFoundException) {
+ throw (NotFoundException)e;
+ } else if (e instanceof BadRequestException) {
+ throw (BadRequestException)e;
} else {
LOG.error("Error while processing REST request", e);
throw new WebApplicationException(e,
@@ -303,6 +192,11 @@ public class TimelineReaderWebServices {
/**
* Return the description of the timeline reader web services.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ *
+ * @return information about the cluster including timeline version.
*/
@GET
@Produces(MediaType.APPLICATION_JSON)
@@ -314,11 +208,188 @@ public class TimelineReaderWebServices {
}
/**
+ * Return a single entity for a given entity type and UID which is a delimited
+ * string containing clusterid, userid, flow name, flowrun id and app id.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param uId a delimited string containing clusterid, userid, flow name,
+ * flowrun id and app id which are extracted from UID and then used to
+ * query backend(Mandatory path param).
+ * @param limit Number of entities to return(Optional query param).
+ * @param createdTimeStart If specified, matched entities should not be
+ * created before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched entities should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched entities should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched entities should be related to
+ * given entities associated with a entity type. relatesto is a comma
+ * separated list in the format [entitytype]:[entityid1]:[entityid2]...
+ * (Optional query param).
+ * @param infofilters If specified, matched entities should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched entities should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched entities should contain the
+ * given metrics. This is represented as
+ * metricfilters=metricid1, metricid2... (Optional query param).
+ * @param eventfilters If specified, matched entities should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the entity object to retrieve, see
+ * {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type, id and created time is returned
+ * (Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances of the given entity type is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request or UID is incorrect,
+ * HTTP 400(Bad Request) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
+ */
+ @GET
+ @Path("/app-uid/{uid}/entities/{entitytype}")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getEntities(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("uid") String uId,
+ @PathParam("entitytype") String entityType,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ Set<TimelineEntity> entities = null;
+ try {
+ TimelineReaderContext context =
+ TimelineUIDConverter.APPLICATION_UID.decodeUID(uId);
+ if (context == null) {
+ throw new BadRequestException("Incorrect UID " + uId);
+ }
+ entities = timelineReaderManager.getEntities(
+ TimelineReaderWebServicesUtils.parseStr(context.getUserId()),
+ TimelineReaderWebServicesUtils.parseStr(context.getClusterId()),
+ TimelineReaderWebServicesUtils.parseStr(context.getFlowName()),
+ context.getFlowRunId(),
+ TimelineReaderWebServicesUtils.parseStr(context.getAppId()),
+ TimelineReaderWebServicesUtils.parseStr(entityType),
+ TimelineReaderWebServicesUtils.parseLongStr(limit),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
+ TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
+ relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
+ isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValueObj(
+ infofilters, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValueStr(
+ conffilters, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseValuesStr(
+ metricfilters, COMMA_DELIMITER),
+ TimelineReaderWebServicesUtils.parseValuesStr(
+ eventfilters, COMMA_DELIMITER),
+ TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
+ } catch (Exception e) {
+ handleException(e, url, startTime,
+ "createdTime start/end or limit or flowrunid");
+ }
+ long endTime = Time.monotonicNow();
+ if (entities == null) {
+ entities = Collections.emptySet();
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entities;
+ }
+
+ /**
* Return a set of entities that match the given parameters. Cluster ID is not
- * provided by client so default cluster ID has to be taken.
+ * provided by client so default cluster ID has to be taken. If userid, flow
+ * name and flowrun id which are optional query parameters are not specified,
+ * they will be queried based on app id and default cluster id from the flow
+ * context information stored in underlying storage implementation. If number
+ * of matching entities are more than the limit, most recent entities till the
+ * limit is reached, will be returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param appId Application id to which the entities to be queried belong to(
+ * Mandatory path param).
+ * @param entityType Type of entities(Mandatory path param).
+ * @param userId User id which should match for the entities(Optional query
+ * param)
+ * @param flowName Flow name which should match for the entities(Optional
+ * query param).
+ * @param flowRunId Run id which should match for the entities(Optional query
+ * param).
+ * @param limit Number of entities to return(Optional query param).
+ * @param createdTimeStart If specified, matched entities should not be
+ * created before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched entities should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched entities should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched entities should be related to
+ * given entities associated with a entity type. relatesto is a comma
+ * separated list in the format [entitytype]:[entityid1]:[entityid2]...
+ * (Optional query param).
+ * @param infofilters If specified, matched entities should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched entities should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched entities should contain the
+ * given metrics. This is represented as
+ * metricfilters=metricid1, metricid2... (Optional query param).
+ * @param eventfilters If specified, matched entities should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the entity object to retrieve, see
+ * {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type, id, created time is returned
+ * (Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances of the given entity type is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow context information cannot be retrieved, HTTP 404(Not Found)
+ * is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/entities/{appid}/{entitytype}/")
+ @Path("/apps/{appid}/entities/{entitytype}")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getEntities(
@Context HttpServletRequest req,
@@ -337,7 +408,7 @@ public class TimelineReaderWebServices {
@QueryParam("conffilters") String conffilters,
@QueryParam("metricfilters") String metricfilters,
@QueryParam("eventfilters") String eventfilters,
- @QueryParam("fields") String fields) {
+ @QueryParam("fields") String fields) {
return getEntities(req, res, null, appId, entityType, userId, flowName,
flowRunId, limit, createdTimeStart, createdTimeEnd, relatesTo,
isRelatedTo, infofilters, conffilters, metricfilters, eventfilters,
@@ -345,10 +416,69 @@ public class TimelineReaderWebServices {
}
/**
- * Return a set of entities that match the given parameters.
+ * Return a set of entities that match the given parameters. If userid, flow
+ * name and flowrun id which are optional query parameters are not specified,
+ * they will be queried based on app id and cluster id from the flow context
+ * information stored in underlying storage implementation. If number of
+ * matching entities are more than the limit, most recent entities till the
+ * limit is reached, will be returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the entities to be queried belong to(
+ * Mandatory path param).
+ * @param appId Application id to which the entities to be queried belong to(
+ * Mandatory path param).
+ * @param entityType Type of entities(Mandatory path param).
+ * @param userId User id which should match for the entities(Optional query
+ * param)
+ * @param flowName Flow name which should match for the entities(Optional
+ * query param).
+ * @param flowRunId Run id which should match for the entities(Optional query
+ * param).
+ * @param limit Number of entities to return(Optional query param).
+ * @param createdTimeStart If specified, matched entities should not be
+ * created before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched entities should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched entities should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched entities should be related to
+ * given entities associated with a entity type. relatesto is a comma
+ * separated list in the format [entitytype]:[entityid1]:[entityid2]...
+ * (Optional query param).
+ * @param infofilters If specified, matched entities should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched entities should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched entities should contain the
+ * given metrics. This is represented as
+ * metricfilters=metricid1, metricid2... (Optional query param).
+ * @param eventfilters If specified, matched entities should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the entity object to retrieve, see
+ * {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type, id, created time is returned
+ * (Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances of the given entity type is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow context information cannot be retrieved, HTTP 404(Not Found)
+ * is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/entities/{clusterid}/{appid}/{entitytype}/")
+ @Path("/clusters/{clusterid}/apps/{appid}/entities/{entitytype}")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getEntities(
@Context HttpServletRequest req,
@@ -372,25 +502,39 @@ public class TimelineReaderWebServices {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
- UserGroupInformation callerUGI = getUser(req);
- LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
- parseStr(userId), parseStr(clusterId), parseStr(flowName),
- parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
- parseLongStr(limit), parseLongStr(createdTimeStart),
- parseLongStr(createdTimeEnd),
- parseKeyStrValuesStr(relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
- parseKeyStrValuesStr(isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
- parseKeyStrValueObj(infofilters, COMMA_DELIMITER, COLON_DELIMITER),
- parseKeyStrValueStr(conffilters, COMMA_DELIMITER, COLON_DELIMITER),
- parseValuesStr(metricfilters, COMMA_DELIMITER),
- parseValuesStr(eventfilters, COMMA_DELIMITER),
- parseFieldsStr(fields, COMMA_DELIMITER));
+ TimelineReaderWebServicesUtils.parseStr(userId),
+ TimelineReaderWebServicesUtils.parseStr(clusterId),
+ TimelineReaderWebServicesUtils.parseStr(flowName),
+ TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
+ TimelineReaderWebServicesUtils.parseStr(appId),
+ TimelineReaderWebServicesUtils.parseStr(entityType),
+ TimelineReaderWebServicesUtils.parseLongStr(limit),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
+ TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
+ relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
+ isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValueObj(
+ infofilters, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValueStr(
+ conffilters, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseValuesStr(
+ metricfilters, COMMA_DELIMITER),
+ TimelineReaderWebServicesUtils.parseValuesStr(
+ eventfilters, COMMA_DELIMITER),
+ TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime,
"createdTime start/end or limit or flowrunid");
@@ -405,11 +549,110 @@ public class TimelineReaderWebServices {
}
/**
+ * Return a single entity for given UID which is a delimited string containing
+ * clusterid, userid, flow name, flowrun id, app id, entity type and entityid.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param uId a delimited string containing clusterid, userid, flow name,
+ * flowrun id, app id, entity type and entity id which are extracted from
+ * UID and then used to query backend(Mandatory path param).
+ * @param fields Specifies which fields of the entity object to retrieve, see
+ * {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type, id, created time is returned
+ * (Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link TimelineEntity} instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request or UID is incorrect,
+ * HTTP 400(Bad Request) is returned.
+ * If entity for the given entity id cannot be found, HTTP 404(Not Found)
+ * is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
+ */
+ @GET
+ @Path("/entity-uid/{uid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getEntity(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("uid") String uId,
+ @QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ TimelineEntity entity = null;
+ try {
+ TimelineReaderContext context =
+ TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uId);
+ if (context == null) {
+ throw new BadRequestException("Incorrect UID " + uId);
+ }
+ entity = timelineReaderManager.getEntity(context.getUserId(),
+ context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId(), context.getEntityType(), context.getEntityId(),
+ TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
+ } catch (Exception e) {
+ handleException(e, url, startTime, "flowrunid");
+ }
+ long endTime = Time.monotonicNow();
+ if (entity == null) {
+ LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
+ (endTime - startTime) + " ms.)");
+ throw new NotFoundException("Timeline entity with uid: " + uId +
+ "is not found");
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entity;
+ }
+
+ /**
* Return a single entity of the given entity type and Id. Cluster ID is not
- * provided by client so default cluster ID has to be taken.
+ * provided by client so default cluster ID has to be taken. If userid, flow
+ * name and flowrun id which are optional query parameters are not specified,
+ * they will be queried based on app id and default cluster id from the flow
+ * context information stored in underlying storage implementation.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param appId Application id to which the entity to be queried belongs to(
+ * Mandatory path param).
+ * @param entityType Type of entity(Mandatory path param).
+ * @param entityId Id of the entity to be fetched(Mandatory path param).
+ * @param userId User id which should match for the entity(Optional query
+ * param).
+ * @param flowName Flow name which should match for the entity(Optional query
+ * param).
+ * @param flowRunId Run id which should match for the entity(Optional query
+ * param).
+ * @param fields Specifies which fields of the entity object to retrieve, see
+ * {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type, id, created time is returned
+ * (Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link TimelineEntity} instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow context information cannot be retrieved or entity for the given
+ * entity id cannot be found, HTTP 404(Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/entity/{appid}/{entitytype}/{entityid}/")
+ @Path("/apps/{appid}/entities/{entitytype}/{entityid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getEntity(
@Context HttpServletRequest req,
@@ -426,10 +669,42 @@ public class TimelineReaderWebServices {
}
/**
- * Return a single entity of the given entity type and Id.
+ * Return a single entity of the given entity type and Id. If userid, flowname
+ * and flowrun id which are optional query parameters are not specified, they
+ * will be queried based on app id and cluster id from the flow context
+ * information stored in underlying storage implementation.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the entity to be queried belongs to(
+ * Mandatory path param).
+ * @param appId Application id to which the entity to be queried belongs to(
+ * Mandatory path param).
+ * @param entityType Type of entity(Mandatory path param).
+ * @param entityId Id of the entity to be fetched(Mandatory path param).
+ * @param userId User id which should match for the entity(Optional query
+ * param).
+ * @param flowName Flow name which should match for the entity(Optional query
+ * param).
+ * @param flowRunId Run id which should match for the entity(Optional query
+ * param).
+ * @param fields Specifies which fields of the entity object to retrieve, see
+ * {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type, id and created time is returned
+ * (Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link TimelineEntity} instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow context information cannot be retrieved or entity for the given
+ * entity id cannot be found, HTTP 404(Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/entity/{clusterid}/{appid}/{entitytype}/{entityid}/")
+ @Path("/clusters/{clusterid}/apps/{appid}/entities/{entitytype}/{entityid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getEntity(
@Context HttpServletRequest req,
@@ -445,17 +720,25 @@ public class TimelineReaderWebServices {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
- UserGroupInformation callerUGI = getUser(req);
- LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
+ String type = TimelineReaderWebServicesUtils.parseStr(entityType);
+ String id = TimelineReaderWebServicesUtils.parseStr(entityId);
try {
entity = timelineReaderManager.getEntity(
- parseStr(userId), parseStr(clusterId), parseStr(flowName),
- parseLongStr(flowRunId), parseStr(appId), parseStr(entityType),
- parseStr(entityId), parseFieldsStr(fields, COMMA_DELIMITER));
+ TimelineReaderWebServicesUtils.parseStr(userId),
+ TimelineReaderWebServicesUtils.parseStr(clusterId),
+ TimelineReaderWebServicesUtils.parseStr(flowName),
+ TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
+ TimelineReaderWebServicesUtils.parseStr(appId), type, id,
+ TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -463,8 +746,8 @@ public class TimelineReaderWebServices {
if (entity == null) {
LOG.info("Processed URL " + url + " but entity not found" + " (Took " +
(endTime - startTime) + " ms.)");
- throw new NotFoundException("Timeline entity {id: " + parseStr(entityId) +
- ", type: " + parseStr(entityType) + " } is not found");
+ throw new NotFoundException("Timeline entity {id: " + id + ", type: " +
+ type + " } is not found");
}
LOG.info("Processed URL " + url +
" (Took " + (endTime - startTime) + " ms.)");
@@ -472,50 +755,155 @@ public class TimelineReaderWebServices {
}
/**
- * Return a single flow run for the given user, flow id and run id.
+ * Return a single flow run for given UID which is a delimited string
+ * containing clusterid, userid, flow name and flowrun id.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param uId a delimited string containing clusterid, userid, flow name and
+ * flowrun id which are extracted from UID and then used to query backend
+ * (Mandatory path param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link FlowRunEntity} instance is returned. By default, all metrics for
+ * the flow run will be returned.
+ * On failures,
+ * If any problem occurs in parsing request or UID is incorrect,
+ * HTTP 400(Bad Request) is returned.
+ * If flow run for the given flow run id cannot be found, HTTP 404
+ * (Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
+ */
+ @GET
+ @Path("/run-uid/{uid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getFlowRun(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("uid") String uId) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ TimelineEntity entity = null;
+ try {
+ TimelineReaderContext context =
+ TimelineUIDConverter.FLOWRUN_UID.decodeUID(uId);
+ if (context == null) {
+ throw new BadRequestException("Incorrect UID " + uId);
+ }
+ entity = timelineReaderManager.getEntity(context.getUserId(),
+ context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
+ null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
+ } catch (Exception e) {
+ handleException(e, url, startTime, "flowrunid");
+ }
+ long endTime = Time.monotonicNow();
+ if (entity == null) {
+ LOG.info("Processed URL " + url + " but flowrun not found (Took " +
+ (endTime - startTime) + " ms.)");
+ throw new NotFoundException("Flowrun with uid: " + uId + "is not found");
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entity;
+ }
+
+ /**
+ * Return a single flow run for the given user, flow name and run id.
* Cluster ID is not provided by client so default cluster ID has to be taken.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param userId User id representing the user who ran the flow run(Mandatory
+ * path param).
+ * @param flowName Flow name to which the flow run to be queried belongs to(
+ * Mandatory path param).
+ * @param flowRunId Id of the flow run to be queried(Mandatory path param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link FlowRunEntity} instance is returned. By default, all metrics for
+ * the flow run will be returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow run for the given flow run id cannot be found, HTTP 404
+ * (Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowrun/{userid}/{flowname}/{flowrunid}/")
+ @Path("/users/{userid}/flows/{flowname}/runs/{flowrunid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getFlowRun(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("userid") String userId,
@PathParam("flowname") String flowName,
- @PathParam("flowrunid") String flowRunId,
- @QueryParam("fields") String fields) {
- return getFlowRun(req, res, userId, null, flowName, flowRunId, fields);
+ @PathParam("flowrunid") String flowRunId) {
+ return getFlowRun(req, res, null, userId, flowName, flowRunId);
}
/**
- * Return a single flow run for the given user, cluster, flow id and run id.
+ * Return a single flow run for the given user, cluster, flow name and run id.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the flow run to be queried belong to(
+ * Mandatory path param).
+ * @param userId User id representing the user who ran the flow run(Mandatory
+ * path param).
+ * @param flowName Flow name to which the flow run to be queried belongs to(
+ * Mandatory path param).
+ * @param flowRunId Id of the flow run to be queried(Mandatory path param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link FlowRunEntity} instance is returned. By default, all metrics for
+ * the flow run will be returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow run for the given flow run id cannot be found, HTTP 404
+ * (Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowrun/{userid}/{clusterid}/{flowname}/{flowrunid}/")
+ @Path("/clusters/{clusterid}/users/{userid}/flows/{flowname}/"
+ + "runs/{flowrunid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getFlowRun(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
- @PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
+ @PathParam("userid") String userId,
@PathParam("flowname") String flowName,
- @PathParam("flowrunid") String flowRunId,
- @QueryParam("fields") String fields) {
+ @PathParam("flowrunid") String flowRunId) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
- UserGroupInformation callerUGI = getUser(req);
- LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
try {
- entity = timelineReaderManager.getEntity(parseStr(userId),
- parseStr(clusterId), parseStr(flowName), parseLongStr(flowRunId), null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), null,
- parseFieldsStr(fields, COMMA_DELIMITER));
+ entity = timelineReaderManager.getEntity(
+ TimelineReaderWebServicesUtils.parseStr(userId),
+ TimelineReaderWebServicesUtils.parseStr(clusterId),
+ TimelineReaderWebServicesUtils.parseStr(flowName),
+ TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
+ null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null);
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -523,8 +911,10 @@ public class TimelineReaderWebServices {
if (entity == null) {
LOG.info("Processed URL " + url + " but flowrun not found (Took " +
(endTime - startTime) + " ms.)");
- throw new NotFoundException("Flow run {flow id: " + parseStr(flowName) +
- ", run id: " + parseLongStr(flowRunId) + " } is not found");
+ throw new NotFoundException("Flow run {flow name: " +
+ TimelineReaderWebServicesUtils.parseStr(flowName) + ", run id: " +
+ TimelineReaderWebServicesUtils.parseLongStr(flowRunId) +
+ " } is not found");
}
LOG.info("Processed URL " + url +
" (Took " + (endTime - startTime) + " ms.)");
@@ -532,11 +922,112 @@ public class TimelineReaderWebServices {
}
/**
- * Return a set of flows runs for the given user and flow id.
+ * Return a list of flow runs for given UID which is a delimited string
+ * containing clusterid, userid and flow name.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param uId a delimited string containing clusterid, userid, and flow name
+ * which are extracted from UID and then used to query backend(Mandatory
+ * path param).
+ * @param flowName Flow name to which the flow runs to be queried belongs to(
+ * Mandatory path param).
+ * @param limit Number of flow runs to return(Optional query param).
+ * @param createdTimeStart If specified, matched flow runs should not be
+ * created before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched flow runs should not be created
+ * after this timestamp(Optional query param).
+ * @param fields Specifies which fields to retrieve, see {@link Field}.
+ * All fields will be retrieved if fields=ALL. Fields other than METRICS
+ * have no meaning for this REST endpoint. If not specified, all fields
+ * other than metrics are returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * set of {@link FlowRunEntity} instances for the given flow are returned.
+ * On failures,
+ * If any problem occurs in parsing request or UID is incorrect,
+ * HTTP 400(Bad Request) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
+ */
+ @GET
+ @Path("/flow-uid/{uid}/runs/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlowRuns(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("uid") String uId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ Set<TimelineEntity> entities = null;
+ try {
+ TimelineReaderContext context =
+ TimelineUIDConverter.FLOW_UID.decodeUID(uId);
+ if (context == null) {
+ throw new BadRequestException("Incorrect UID " + uId);
+ }
+ entities = timelineReaderManager.getEntities(context.getUserId(),
+ context.getClusterId(), context.getFlowName(), null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(),
+ TimelineReaderWebServicesUtils.parseLongStr(limit),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
+ null, null, null, null, null, null, TimelineReaderWebServicesUtils.
+ parseFieldsStr(fields, COMMA_DELIMITER));
+ } catch (Exception e) {
+ handleException(e, url, startTime, "createdTime start/end or limit");
+ }
+ long endTime = Time.monotonicNow();
+ if (entities == null) {
+ entities = Collections.emptySet();
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entities;
+ }
+
+ /**
+ * Return a set of flows runs for the given user and flow name.
* Cluster ID is not provided by client so default cluster ID has to be taken.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param userId User id representing the user who ran the flow runs(
+ * Mandatory path param)
+ * @param flowName Flow name to which the flow runs to be queried belongs to(
+ * Mandatory path param).
+ * @param limit Number of flow runs to return(Optional query param).
+ * @param createdTimeStart If specified, matched flow runs should not be
+ * created before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched flow runs should not be created
+ * after this timestamp(Optional query param).
+ * @param fields Specifies which fields to retrieve, see {@link Field}.
+ * All fields will be retrieved if fields=ALL. Fields other than METRICS
+ * have no meaning for this REST endpoint. If not specified, all fields
+ * other than metrics are returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * set of {@link FlowRunEntity} instances for the given flow are returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowruns/{userid}/{flowname}/")
+ @Path("/users/{userid}/flows/{flowname}/runs/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRuns(
@Context HttpServletRequest req,
@@ -547,21 +1038,47 @@ public class TimelineReaderWebServices {
@QueryParam("createdtimestart") String createdTimeStart,
@QueryParam("createdtimeend") String createdTimeEnd,
@QueryParam("fields") String fields) {
- return getFlowRuns(req, res, userId, null, flowName, limit, createdTimeStart,
- createdTimeEnd, fields);
+ return getFlowRuns(req, res, null, userId, flowName, limit,
+ createdTimeStart, createdTimeEnd, fields);
}
/**
- * Return a set of flow runs for the given user, cluster and flow id.
+ * Return a set of flows runs for the given cluster, user and flow name.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the flow runs to be queried belong to(
+ * Mandatory path param).
+ * @param userId User id representing the user who ran the flow runs(
+ * Mandatory path param)
+ * @param flowName Flow name to which the flow runs to be queried belongs to(
+ * Mandatory path param).
+ * @param limit Number of flow runs to return(Optional query param).
+ * @param createdTimeStart If specified, matched flow runs should not be
+ * created before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched flow runs should not be created
+ * after this timestamp(Optional query param).
+ * @param fields Specifies which fields to retrieve, see {@link Field}.
+ * All fields will be retrieved if fields=ALL. Fields other than METRICS
+ * have no meaning for this REST endpoint. If not specified, all fields
+ * other than metrics are returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * set of {@link FlowRunEntity} instances for the given flow are returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowruns/{userid}/{clusterid}/{flowname}/")
+ @Path("/clusters/{clusterid}/users/{userid}/flows/{flowname}/runs/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRuns(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
- @PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
+ @PathParam("userid") String userId,
@PathParam("flowname") String flowName,
@QueryParam("limit") String limit,
@QueryParam("createdtimestart") String createdTimeStart,
@@ -570,18 +1087,24 @@ public class TimelineReaderWebServices {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
- UserGroupInformation callerUGI = getUser(req);
- LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
Set<TimelineEntity> entities = null;
try {
entities = timelineReaderManager.getEntities(
- parseStr(userId), parseStr(clusterId), parseStr(flowName), null, null,
- TimelineEntityType.YARN_FLOW_RUN.toString(), parseLongStr(limit),
- parseLongStr(createdTimeStart), parseLongStr(createdTimeEnd),
- null, null, null, null, null, null,
+ TimelineReaderWebServicesUtils.parseStr(userId),
+ TimelineReaderWebServicesUtils.parseStr(clusterId),
+ TimelineReaderWebServicesUtils.parseStr(flowName), null, null,
+ TimelineEntityType.YARN_FLOW_RUN.toString(),
+ TimelineReaderWebServicesUtils.parseLongStr(limit),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
+ null, null, null, null, null, null, TimelineReaderWebServicesUtils.
parseFieldsStr(fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "createdTime start/end or limit");
@@ -596,20 +1119,35 @@ public class TimelineReaderWebServices {
}
/**
- * Return a list of flows. Cluster ID is not provided by client so default
- * cluster ID has to be taken. daterange, if specified is given as
- * "[startdate]-[enddate]"(i.e. start and end date separated by -) or
- * single date. Dates are interpreted in yyyyMMdd format and are assumed to
- * be in GMT. If a single date is specified, all flows active on that date are
- * returned. If both startdate and enddate is given, all flows active between
- * start and end date will be returned. If only startdate is given, flows
- * active on and after startdate are returned. If only enddate is given, flows
- * active on and before enddate are returned.
- * For example :
- * "daterange=20150711" returns flows active on 20150711.
- * "daterange=20150711-20150714" returns flows active between these 2 dates.
- * "daterange=20150711-" returns flows active on and after 20150711.
- * "daterange=-20150711" returns flows active on and before 20150711.
+ * Return a list of active flows. Cluster ID is not provided by client so
+ * default cluster ID has to be taken.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param limit Number of flows to return(Optional query param).
+ * @param dateRange If specified is given as "[startdate]-[enddate]"(i.e.
+ * start and end date separated by "-") or single date. Dates are
+ * interpreted in yyyyMMdd format and are assumed to be in GMT(Optional
+ * query param).
+ * If a single date is specified, all flows active on that date are
+ * returned. If both startdate and enddate is given, all flows active
+ * between start and end date will be returned. If only startdate is
+ * given, flows active on and after startdate are returned. If only
+ * enddate is given, flows active on and before enddate are returned.
+ * For example :
+ * "daterange=20150711" returns flows active on 20150711.
+ * "daterange=20150711-20150714" returns flows active between these
+ * 2 dates.
+ * "daterange=20150711-" returns flows active on and after 20150711.
+ * "daterange=-20150711" returns flows active on and before 20150711.
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * set of {@link FlowActivityEntity} instances are returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
@Path("/flows/")
@@ -618,41 +1156,58 @@ public class TimelineReaderWebServices {
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@QueryParam("limit") String limit,
- @QueryParam("daterange") String dateRange,
- @QueryParam("fields") String fields) {
- return getFlows(req, res, null, limit, dateRange, fields);
+ @QueryParam("daterange") String dateRange) {
+ return getFlows(req, res, null, limit, dateRange);
}
/**
- * Return a list of flows for a given cluster id. daterange, if specified is
- * given as "[startdate]-[enddate]"(i.e. start and end date separated by -) or
- * single date. Dates are interpreted in yyyyMMdd format and are assumed to
- * be in GMT. If a single date is specified, all flows active on that date are
- * returned. If both startdate and enddate is given, all flows active between
- * start and end date will be returned. If only startdate is given, flows
- * active on and after startdate are returned. If only enddate is given, flows
- * active on and before enddate are returned.
- * For example :
- * "daterange=20150711" returns flows active on 20150711.
- * "daterange=20150711-20150714" returns flows active between these 2 dates.
- * "daterange=20150711-" returns flows active on and after 20150711.
- * "daterange=-20150711" returns flows active on and before 20150711.
+ * Return a list of active flows for a given cluster id.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the flows to be queried belong to(
+ * Mandatory path param).
+ * @param limit Number of flows to return(Optional query param).
+ * @param dateRange If specified is given as "[startdate]-[enddate]"(i.e.
+ * start and end date separated by "-") or single date. Dates are
+ * interpreted in yyyyMMdd format and are assumed to be in GMT(Optional
+ * query param).
+ * If a single date is specified, all flows active on that date are
+ * returned. If both startdate and enddate is given, all flows active
+ * between start and end date will be returned. If only startdate is
+ * given, flows active on and after startdate are returned. If only
+ * enddate is given, flows active on and before enddate are returned.
+ * For example :
+ * "daterange=20150711" returns flows active on 20150711.
+ * "daterange=20150711-20150714" returns flows active between these
+ * 2 dates.
+ * "daterange=20150711-" returns flows active on and after 20150711.
+ * "daterange=-20150711" returns flows active on and before 20150711.
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * set of {@link FlowActivityEntity} instances are returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flows/{clusterid}/")
+ @Path("/clusters/{clusterid}/flows/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlows(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
@PathParam("clusterid") String clusterId,
@QueryParam("limit") String limit,
- @QueryParam("daterange") String dateRange,
- @QueryParam("fields") String fields) {
+ @QueryParam("daterange") String dateRange) {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
- UserGroupInformation callerUGI = getUser(req);
- LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
@@ -660,10 +1215,10 @@ public class TimelineReaderWebServices {
try {
DateRange range = parseDateRange(dateRange);
entities = timelineReaderManager.getEntities(
- null, parseStr(clusterId), null, null, null,
- TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), parseLongStr(limit),
- range.dateStart, range.dateEnd, null, null, null, null, null, null,
- parseFieldsStr(fields, COMMA_DELIMITER));
+ null, TimelineReaderWebServicesUtils.parseStr(clusterId), null, null,
+ null, TimelineEntityType.YARN_FLOW_ACTIVITY.toString(),
+ TimelineReaderWebServicesUtils.parseLongStr(limit), range.dateStart,
+ range.dateEnd, null, null, null, null, null, null, null);
} catch (Exception e) {
handleException(e, url, startTime, "limit");
}
@@ -677,11 +1232,105 @@ public class TimelineReaderWebServices {
}
/**
- * Return a single app for given app id. Cluster ID is not provided by
- * client so default cluster ID has to be taken.
+ * Return a single app for given UID which is a delimited string containing
+ * clusterid, userid, flow name, flowrun id and app id.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param uId a delimited string containing clusterid, userid, flow name, flow
+ * run id and app id which are extracted from UID and then used to query
+ * backend(Mandatory path param).
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link TimelineEntity} instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request or UID is incorrect,
+ * HTTP 400(Bad Request) is returned.
+ * If app for the given app id cannot be found, HTTP 404(Not Found) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
+ */
+ @GET
+ @Path("/app-uid/{uid}/")
+ @Produces(MediaType.APPLICATION_JSON)
+ public TimelineEntity getApp(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("uid") String uId,
+ @QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ TimelineEntity entity = null;
+ try {
+ TimelineReaderContext context =
+ TimelineUIDConverter.APPLICATION_UID.decodeUID(uId);
+ if (context == null) {
+ throw new BadRequestException("Incorrect UID " + uId);
+ }
+ entity = timelineReaderManager.getEntity(context.getUserId(),
+ context.getClusterId(), context.getFlowName(), context.getFlowRunId(),
+ context.getAppId(), TimelineEntityType.YARN_APPLICATION.toString(),
+ null, TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
+ } catch (Exception e) {
+ handleException(e, url, startTime, "flowrunid");
+ }
+ long endTime = Time.monotonicNow();
+ if (entity == null) {
+ LOG.info("Processed URL " + url + " but app not found" + " (Took " +
+ (endTime - startTime) + " ms.)");
+ throw new NotFoundException("App with uid " + uId + " not found");
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entity;
+ }
+
+ /**
+ * Return a single app for given app id. Cluster ID is not provided by client
+ * client so default cluster ID has to be taken. If userid, flow name and flow
+ * run id which are optional query parameters are not specified, they will be
+ * queried based on app id and cluster id from the flow context information
+ * stored in underlying storage implementation.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param appId Application id to be queried(Mandatory path param).
+ * @param flowName Flow name which should match for the app(Optional query
+ * param).
+ * @param flowRunId Run id which should match for the app(Optional query
+ * param).
+ * @param userId User id which should match for the app(Optional query param).
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link TimelineEntity} instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow context information cannot be retrieved or app for the given
+ * app id cannot be found, HTTP 404(Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/app/{appid}/")
+ @Path("/apps/{appid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getApp(
@Context HttpServletRequest req,
@@ -695,10 +1344,38 @@ public class TimelineReaderWebServices {
}
/**
- * Return a single app for given cluster id and app id.
+ * Return a single app for given cluster id and app id. If userid, flow name
+ * and flowrun id which are optional query parameters are not specified, they
+ * will be queried based on app id and cluster id from the flow context
+ * information stored in underlying storage implementation.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the app to be queried belong to(
+ * Mandatory path param).
+ * @param appId Application id to be queried(Mandatory path param).
+ * @param flowName Flow name which should match for the app(Optional query
+ * param).
+ * @param flowRunId Run id which should match for the app(Optional query
+ * param).
+ * @param userId User id which should match for the app(Optional query param).
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing a
+ * {@link TimelineEntity} instance is returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * If flow context information cannot be retrieved or app for the given
+ * app id cannot be found, HTTP 404(Not Found) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/app/{clusterid}/{appid}/")
+ @Path("/clusters/{clusterid}/apps/{appid}/")
@Produces(MediaType.APPLICATION_JSON)
public TimelineEntity getApp(
@Context HttpServletRequest req,
@@ -712,17 +1389,24 @@ public class TimelineReaderWebServices {
String url = req.getRequestURI() +
(req.getQueryString() == null ? "" :
QUERY_STRING_SEP + req.getQueryString());
- UserGroupInformation callerUGI = getUser(req);
- LOG.info("Received URL " + url + " from user " + getUserName(callerUGI));
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
long startTime = Time.monotonicNow();
init(res);
TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
TimelineEntity entity = null;
try {
- entity = timelineReaderManager.getEntity(parseStr(userId),
- parseStr(clusterId), parseStr(flowName), parseLongStr(flowRunId),
- parseStr(appId), TimelineEntityType.YARN_APPLICATION.toString(), null,
- parseFieldsStr(fields, COMMA_DELIMITER));
+ entity = timelineReaderManager.getEntity(
+ TimelineReaderWebServicesUtils.parseStr(userId),
+ TimelineReaderWebServicesUtils.parseStr(clusterId),
+ TimelineReaderWebServicesUtils.parseStr(flowName),
+ TimelineReaderWebServicesUtils.parseLongStr(flowRunId),
+ TimelineReaderWebServicesUtils.parseStr(appId),
+ TimelineEntityType.YARN_APPLICATION.toString(), null,
+ TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
} catch (Exception e) {
handleException(e, url, startTime, "flowrunid");
}
@@ -738,13 +1422,180 @@ public class TimelineReaderWebServices {
}
/**
- * Return a list of apps for given user, flow id and flow run id. Cluster ID
+ * Return a list of apps for given UID which is a delimited string containing
+ * clusterid, userid, flow name and flowrun id. If number of matching apps are
+ * more than the limit, most recent apps till the limit is reached, will be
+ * returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param uId a delimited string containing clusterid, userid, flow name and
+ * flowrun id which are extracted from UID and then used to query backend.
+ * (Mandatory path param).
+ * @param limit Number of apps to return(Optional query param).
+ * @param createdTimeStart If specified, matched apps should not be created
+ * before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched apps should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched apps should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched apps should be related to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param infofilters If specified, matched apps should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched apps should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched apps should contain the given
+ * metrics. This is represented as metricfilters=metricid1, metricid2...
+ * (Optional query param).
+ * @param eventfilters If specified, matched apps should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances representing apps is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request or UID is incorrect,
+ * HTTP 400(Bad Request) is returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
+ */
+ @GET
+ @Path("/run-uid/{uid}/apps")
+ @Produces(MediaType.APPLICATION_JSON)
+ public Set<TimelineEntity> getFlowRunApps(
+ @Context HttpServletRequest req,
+ @Context HttpServletResponse res,
+ @PathParam("uid") String uId,
+ @QueryParam("limit") String limit,
+ @QueryParam("createdtimestart") String createdTimeStart,
+ @QueryParam("createdtimeend") String createdTimeEnd,
+ @QueryParam("relatesto") String relatesTo,
+ @QueryParam("isrelatedto") String isRelatedTo,
+ @QueryParam("infofilters") String infofilters,
+ @QueryParam("conffilters") String conffilters,
+ @QueryParam("metricfilters") String metricfilters,
+ @QueryParam("eventfilters") String eventfilters,
+ @QueryParam("fields") String fields) {
+ String url = req.getRequestURI() +
+ (req.getQueryString() == null ? "" :
+ QUERY_STRING_SEP + req.getQueryString());
+ UserGroupInformation callerUGI =
+ TimelineReaderWebServicesUtils.getUser(req);
+ LOG.info("Received URL " + url + " from user " +
+ TimelineReaderWebServicesUtils.getUserName(callerUGI));
+ long startTime = Time.monotonicNow();
+ init(res);
+ TimelineReaderManager timelineReaderManager = getTimelineReaderManager();
+ Set<TimelineEntity> entities = null;
+ try {
+ TimelineReaderContext context =
+ TimelineUIDConverter.FLOWRUN_UID.decodeUID(uId);
+ if (context == null) {
+ throw new BadRequestException("Incorrect UID " + uId);
+ }
+ entities = timelineReaderManager.getEntities(
+ TimelineReaderWebServicesUtils.parseStr(context.getUserId()),
+ TimelineReaderWebServicesUtils.parseStr(context.getClusterId()),
+ TimelineReaderWebServicesUtils.parseStr(context.getFlowName()),
+ context.getFlowRunId(),
+ TimelineReaderWebServicesUtils.parseStr(context.getAppId()),
+ TimelineEntityType.YARN_APPLICATION.toString(),
+ TimelineReaderWebServicesUtils.parseLongStr(limit),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeStart),
+ TimelineReaderWebServicesUtils.parseLongStr(createdTimeEnd),
+ TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
+ relatesTo, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValuesStr(
+ isRelatedTo, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValueObj(
+ infofilters, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseKeyStrValueStr(
+ conffilters, COMMA_DELIMITER, COLON_DELIMITER),
+ TimelineReaderWebServicesUtils.parseValuesStr(
+ metricfilters, COMMA_DELIMITER),
+ TimelineReaderWebServicesUtils.parseValuesStr(
+ eventfilters, COMMA_DELIMITER),
+ TimelineReaderWebServicesUtils.parseFieldsStr(
+ fields, COMMA_DELIMITER));
+ } catch (Exception e) {
+ handleException(e, url, startTime,
+ "createdTime start/end or limit or flowrunid");
+ }
+ long endTime = Time.monotonicNow();
+ if (entities == null) {
+ entities = Collections.emptySet();
+ }
+ LOG.info("Processed URL " + url +
+ " (Took " + (endTime - startTime) + " ms.)");
+ return entities;
+ }
+
+ /**
+ * Return a list of apps for given user, flow name and flow run id. Cluster ID
* is not provided by client so default cluster ID has to be taken. If number
* of matching apps are more than the limit, most recent apps till the limit
* is reached, will be returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param userId User id which should match for the apps(Mandatory path param)
+ * @param flowName Flow name which should match for the apps(Mandatory path
+ * param).
+ * @param flowRunId Run id which should match for the apps(Mandatory path
+ * param).
+ * @param limit Number of apps to return(Optional query param).
+ * @param createdTimeStart If specified, matched apps should not be created
+ * before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched apps should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched apps should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched apps should be related to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param infofilters If specified, matched apps should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched apps should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched apps should contain the given
+ * metrics. This is represented as metricfilters=metricid1, metricid2...
+ * (Optional query param).
+ * @param eventfilters If specified, matched apps should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances representing apps is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowrunapps/{userid}/{flowname}/{flowrunid}/")
+ @Path("/users/{userid}/flows/{flowname}/runs/{flowrunid}/apps/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRunApps(
@Context HttpServletRequest req,
@@ -770,18 +1621,67 @@ public class TimelineReaderWebServices {
}
/**
- * Return a list of apps for a given user, cluster id, flow id and flow run
+ * Return a list of apps for a given user, cluster id, flow name and flow run
* id. If number of matching apps are more than the limit, most recent apps
* till the limit is reached, will be returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the apps to be queried belong to
+ * (Mandatory path param).
+ * @param userId User id which should match for the apps(Mandatory path param)
+ * @param flowName Flow name which should match for the apps(Mandatory path
+ * param).
+ * @param flowRunId Run id which should match for the apps(Mandatory path
+ * param).
+ * @param limit Number of apps to return(Optional query param).
+ * @param createdTimeStart If specified, matched apps should not be created
+ * before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched apps should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched apps should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched apps should be related to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param infofilters If specified, matched apps should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched apps should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched apps should contain the given
+ * metrics. This is represented as metricfilters=metricid1, metricid2...
+ * (Optional query param).
+ * @param eventfilters If specified, matched apps should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances representing apps is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowrunapps/{userid}/{clusterid}/{flowname}/{flowrunid}/")
+ @Path("/clusters/{clusterid}/users/{userid}/flows/{flowname}/runs/"
+ + "{flowrunid}/apps/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowRunApps(
@Context HttpServletRequest req,
@Context HttpServletResponse res,
- @PathParam("userid") String userId,
@PathParam("clusterid") String clusterId,
+ @PathParam("userid") String userId,
@PathParam("flowname") String flowName,
@PathParam("flowrunid") String flowRunId,
@QueryParam("limit") String limit,
@@ -802,13 +1702,57 @@ public class TimelineReaderWebServices {
}
/**
- * Return a list of apps for given user and flow id. Cluster ID is not
+ * Return a list of apps for given user and flow name. Cluster ID is not
* provided by client so default cluster ID has to be taken. If number of
* matching apps are more than the limit, most recent apps till the limit is
* reached, will be returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param userId User id which should match for the apps(Mandatory path param)
+ * @param flowName Flow name which should match for the apps(Mandatory path
+ * param).
+ * @param limit Number of apps to return(Optional query param).
+ * @param createdTimeStart If specified, matched apps should not be created
+ * before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched apps should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched apps should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched apps should be related to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param infofilters If specified, matched apps should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched apps should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched apps should contain the given
+ * metrics. This is represented as metricfilters=metricid1, metricid2...
+ * (Optional query param).
+ * @param eventfilters If specified, matched apps should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ * @param fields Specifies which fields of the app entity object to retrieve,
+ * see {@link Field}. All fields will be retrieved if fields=ALL. If not
+ * specified, 4 fields i.e. entity type(equivalent to YARN_APPLICATION),
+ * app id and app created time is returned(Optional query param).
+ *
+ * @return If successful, a HTTP 200(OK) response having a JSON representing
+ * a set of {@link TimelineEntity} instances representing apps is
+ * returned.
+ * On failures,
+ * If any problem occurs in parsing request, HTTP 400(Bad Request) is
+ * returned.
+ * For all other errors while retrieving data, HTTP 500(Internal Server
+ * Error) is returned.
*/
@GET
- @Path("/flowapps/{userid}/{flowname}/")
+ @Path("/users/{userid}/flows/{flowname}/apps/")
@Produces(MediaType.APPLICATION_JSON)
public Set<TimelineEntity> getFlowApps(
@Context HttpServletRequest req,
@@ -832,18 +1776,65 @@ public class TimelineReaderWebServices {
}
/**
- * Return a list of apps for a given user, cluster id and flow id. If number
+ * Return a list of apps for a given user, cluster id and flow name. If number
* of matching apps are more than the limit, most recent apps till the limit
- * is reached, will be returned.
+ * is reached, will be returned. If number of matching apps are more than the
+ * limit, most recent apps till the limit is reached, will be returned.
+ *
+ * @param req Servlet request.
+ * @param res Servlet response.
+ * @param clusterId Cluster id to which the apps to be queried belong to
+ * (Mandatory path param).
+ * @param userId User id which should match for the apps(Mandatory path param)
+ * @param flowName Flow name which should match for the apps(Mandatory path
+ * param).
+ * @param limit Number of apps to return(Optional query param).
+ * @param createdTimeStart If specified, matched apps should not be created
+ * before this timestamp(Optional query param).
+ * @param createdTimeEnd If specified, matched apps should not be created
+ * after this timestamp(Optional query param).
+ * @param relatesTo If specified, matched apps should relate to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param isRelatedTo If specified, matched apps should be related to given
+ * entities associated with a entity type. relatesto is a comma separated
+ * list in the format [entitytype]:[entityid1]:[entityid2]... (Optional
+ * query param).
+ * @param infofilters If specified, matched apps should have exact matches
+ * to the given info represented as key-value pairs. This is represented
+ * as infofilters=info1:value1,info2:value2... (Optional query param).
+ * @param conffilters If specified, matched apps should have exact matches
+ * to the given configs represented as key-value pairs. This is
+ * represented as conffilters=conf1:value1,conf2:value2... (Optional query
+ * param).
+ * @param metricfilters If specified, matched apps should contain the given
+ * metrics. This is represented as metricfilters=metricid1, metricid2...
+ * (Optional query param).
+ * @param eventfilters If specified, matched apps should contain the given
+ * events. This is represented as eventfilters=eventid1, eventid2...
+ *
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: YARN-4200. Refactor reader classes
in storage to nest under hbase specific package name. Contributed by Li Lu.
Posted by gt...@apache.org.
YARN-4200. Refactor reader classes in storage to nest under hbase
specific package name. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/61737325
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/61737325
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/61737325
Branch: refs/heads/YARN-2928
Commit: 61737325ccaaea48e20d2f288c0c9d1b43a8bccb
Parents: ed5f7db
Author: Li Lu <gt...@apache.org>
Authored: Mon Jan 11 18:05:36 2016 -0800
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:22:12 2016 -0700
----------------------------------------------------------------------
.../storage/ApplicationEntityReader.java | 382 --------------
.../storage/FlowActivityEntityReader.java | 163 ------
.../storage/FlowRunEntityReader.java | 225 ---------
.../storage/GenericEntityReader.java | 496 ------------------
.../storage/HBaseTimelineReaderImpl.java | 2 +
.../storage/TimelineEntityReader.java | 274 ----------
.../storage/TimelineEntityReaderFactory.java | 100 ----
.../storage/reader/ApplicationEntityReader.java | 383 ++++++++++++++
.../reader/FlowActivityEntityReader.java | 164 ++++++
.../storage/reader/FlowRunEntityReader.java | 226 +++++++++
.../storage/reader/GenericEntityReader.java | 497 +++++++++++++++++++
.../storage/reader/TimelineEntityReader.java | 274 ++++++++++
.../reader/TimelineEntityReaderFactory.java | 100 ++++
.../storage/reader/package-info.java | 23 +
14 files changed, 1669 insertions(+), 1640 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
deleted file mode 100644
index d812a6c..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/ApplicationEntityReader.java
+++ /dev/null
@@ -1,382 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for application entities that are stored in the
- * application table.
- */
-class ApplicationEntityReader extends GenericEntityReader {
- private static final ApplicationTable APPLICATION_TABLE =
- new ApplicationTable();
-
- public ApplicationEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
- true);
- }
-
- public ApplicationEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
- }
-
- /**
- * Uses the {@link ApplicationTable}.
- */
- protected BaseTable<?> getTable() {
- return APPLICATION_TABLE;
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
- // Fetch all the columns.
- if (fieldsToRetrieve.contains(Field.ALL) &&
- (confsToRetrieve == null ||
- confsToRetrieve.getFilterList().isEmpty()) &&
- (metricsToRetrieve == null ||
- metricsToRetrieve.getFilterList().isEmpty())) {
- return list;
- }
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
- // Events not required.
- if (!fieldsToRetrieve.contains(Field.EVENTS) &&
- !fieldsToRetrieve.contains(Field.ALL) && eventFilters == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.EVENT.getColumnPrefixBytes(""))));
- }
- // info not required.
- if (!fieldsToRetrieve.contains(Field.INFO) &&
- !fieldsToRetrieve.contains(Field.ALL) && infoFilters == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.INFO.getColumnPrefixBytes(""))));
- }
- // is releated to not required.
- if (!fieldsToRetrieve.contains(Field.IS_RELATED_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && isRelatedTo == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
- }
- // relates to not required.
- if (!fieldsToRetrieve.contains(Field.RELATES_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && relatesTo == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- ApplicationColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
- }
- list.addFilter(infoColFamilyList);
- if ((fieldsToRetrieve.contains(Field.CONFIGS) || configFilters != null) ||
- (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty())) {
- FilterList filterCfg =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.CONFIGS.getBytes())));
- if (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty()) {
- filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.CONFIG, confsToRetrieve));
- }
- list.addFilter(filterCfg);
- }
- if ((fieldsToRetrieve.contains(Field.METRICS) || metricFilters != null) ||
- (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty())) {
- FilterList filterMetrics =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(ApplicationColumnFamily.METRICS.getBytes())));
- if (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
- ApplicationColumnPrefix.METRIC, metricsToRetrieve));
- }
- list.addFilter(filterMetrics);
- }
- return list;
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- byte[] rowKey =
- ApplicationRowKey.getRowKey(clusterId, userId, flowName, flowRunId,
- appId);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- get.setFilter(filterList);
- }
- return table.getResult(hbaseConf, conn, get);
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
- if (singleEntityRead) {
- Preconditions.checkNotNull(appId, "appId shouldn't be null");
- } else {
- Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
- }
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- if (singleEntityRead) {
- if (flowName == null || flowRunId == null || userId == null) {
- FlowContext context =
- lookupFlowContext(clusterId, appId, hbaseConf, conn);
- flowName = context.flowName;
- flowRunId = context.flowRunId;
- userId = context.userId;
- }
- }
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
- if (!fieldsToRetrieve.contains(Field.CONFIGS) &&
- confsToRetrieve != null && !confsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.CONFIGS);
- }
- if (!fieldsToRetrieve.contains(Field.METRICS) &&
- metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.METRICS);
- }
- if (!singleEntityRead) {
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
- }
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException {
- Scan scan = new Scan();
- if (flowRunId != null) {
- scan.setRowPrefixFilter(ApplicationRowKey.
- getRowKeyPrefix(clusterId, userId, flowName, flowRunId));
- } else {
- scan.setRowPrefixFilter(ApplicationRowKey.
- getRowKeyPrefix(clusterId, userId, flowName));
- }
- FilterList newList = new FilterList();
- newList.addFilter(new PageFilter(limit));
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- newList.addFilter(filterList);
- }
- scan.setFilter(newList);
- return table.getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
- String entityId = ApplicationColumn.ID.readResult(result).toString();
- entity.setId(entityId);
-
- // fetch created time
- Number createdTime =
- (Number)ApplicationColumn.CREATED_TIME.readResult(result);
- entity.setCreatedTime(createdTime.longValue());
- if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
- entity.getCreatedTime() > createdTimeEnd)) {
- return null;
- }
-
- // fetch modified time
- Number modifiedTime =
- (Number)ApplicationColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
- // fetch is related to entities
- boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, ApplicationColumnPrefix.IS_RELATED_TO,
- true);
- if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), isRelatedTo)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
- entity.getIsRelatedToEntities().clear();
- }
- }
-
- // fetch relates to entities
- boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, ApplicationColumnPrefix.RELATES_TO,
- false);
- if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), relatesTo)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.RELATES_TO)) {
- entity.getRelatesToEntities().clear();
- }
- }
-
- // fetch info
- boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.INFO, false);
- if (checkInfo &&
- !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.INFO)) {
- entity.getInfo().clear();
- }
- }
-
- // fetch configs
- boolean checkConfigs = configFilters != null && configFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- readKeyValuePairs(entity, result, ApplicationColumnPrefix.CONFIG, true);
- if (checkConfigs && !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), configFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.CONFIGS)) {
- entity.getConfigs().clear();
- }
- }
-
- // fetch events
- boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result, true);
- if (checkEvents && !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), eventFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.EVENTS)) {
- entity.getEvents().clear();
- }
- }
-
- // fetch metrics
- boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
- readMetrics(entity, result, ApplicationColumnPrefix.METRIC);
- if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), metricFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.METRICS)) {
- entity.getMetrics().clear();
- }
- }
- return entity;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
deleted file mode 100644
index 7e8d4ba..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowActivityEntityReader.java
+++ /dev/null
@@ -1,163 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTable;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for flow activity entities that are stored in the
- * flow activity table.
- */
-class FlowActivityEntityReader extends TimelineEntityReader {
- private static final FlowActivityTable FLOW_ACTIVITY_TABLE =
- new FlowActivityTable();
-
- public FlowActivityEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, null, null, fieldsToRetrieve, true);
- }
-
- public FlowActivityEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- null, null, fieldsToRetrieve);
- }
-
- /**
- * Uses the {@link FlowActivityTable}.
- */
- @Override
- protected BaseTable<?> getTable() {
- return FLOW_ACTIVITY_TABLE;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() {
- return null;
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- throw new UnsupportedOperationException(
- "we don't support a single entity query");
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException {
- Scan scan = new Scan();
- if (createdTimeBegin == DEFAULT_BEGIN_TIME &&
- createdTimeEnd == DEFAULT_END_TIME) {
- scan.setRowPrefixFilter(FlowActivityRowKey.getRowKeyPrefix(clusterId));
- } else {
- scan.setStartRow(
- FlowActivityRowKey.getRowKeyPrefix(clusterId, createdTimeEnd));
- scan.setStopRow(
- FlowActivityRowKey.getRowKeyPrefix(clusterId,
- (createdTimeBegin <= 0 ? 0: (createdTimeBegin - 1))));
- }
- // use the page filter to limit the result to the page size
- // the scanner may still return more than the limit; therefore we need to
- // read the right number as we iterate
- scan.setFilter(new PageFilter(limit));
- return table.getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- FlowActivityRowKey rowKey = FlowActivityRowKey.parseRowKey(result.getRow());
-
- long time = rowKey.getDayTimestamp();
- String user = rowKey.getUserId();
- String flowName = rowKey.getFlowName();
-
- FlowActivityEntity flowActivity =
- new FlowActivityEntity(clusterId, time, user, flowName);
- // set the id
- flowActivity.setId(flowActivity.getId());
- // get the list of run ids along with the version that are associated with
- // this flow on this day
- Map<String, Object> runIdsMap =
- FlowActivityColumnPrefix.RUN_ID.readResults(result);
- for (Map.Entry<String, Object> e : runIdsMap.entrySet()) {
- Long runId = Long.valueOf(e.getKey());
- String version = (String)e.getValue();
- FlowRunEntity flowRun = new FlowRunEntity();
- flowRun.setUser(user);
- flowRun.setName(flowName);
- flowRun.setRunId(runId);
- flowRun.setVersion(version);
- // set the id
- flowRun.setId(flowRun.getId());
- flowActivity.addFlowRun(flowRun);
- }
-
- return flowActivity;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
deleted file mode 100644
index c9076ee..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/FlowRunEntityReader.java
+++ /dev/null
@@ -1,225 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.PageFilter;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowRunTable;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for flow run entities that are stored in the flow run
- * table.
- */
-class FlowRunEntityReader extends TimelineEntityReader {
- private static final FlowRunTable FLOW_RUN_TABLE = new FlowRunTable();
-
- public FlowRunEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, null, metricsToRetrieve, fieldsToRetrieve, true);
- }
-
- public FlowRunEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- null, metricsToRetrieve, fieldsToRetrieve);
- }
-
- /**
- * Uses the {@link FlowRunTable}.
- */
- @Override
- protected BaseTable<?> getTable() {
- return FLOW_RUN_TABLE;
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(userId, "userId shouldn't be null");
- Preconditions.checkNotNull(flowName, "flowName shouldn't be null");
- if (singleEntityRead) {
- Preconditions.checkNotNull(flowRunId, "flowRunId shouldn't be null");
- }
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn) {
- if (!singleEntityRead) {
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
- if (!fieldsToRetrieve.contains(Field.METRICS) &&
- metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.METRICS);
- }
- }
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
-
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(FlowRunColumnFamily.INFO.getBytes()));
- // Metrics not required.
- if (!singleEntityRead && !fieldsToRetrieve.contains(Field.METRICS) &&
- !fieldsToRetrieve.contains(Field.ALL)) {
- FilterList infoColFamilyList = new FilterList(Operator.MUST_PASS_ONE);
- infoColFamilyList.addFilter(infoColumnFamily);
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- FlowRunColumnPrefix.METRIC.getColumnPrefixBytes(""))));
- list.addFilter(infoColFamilyList);
- }
- if (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- FilterList infoColFamilyList = new FilterList();
- infoColFamilyList.addFilter(infoColumnFamily);
- infoColFamilyList.addFilter(TimelineFilterUtils.createHBaseFilterList(
- FlowRunColumnPrefix.METRIC, metricsToRetrieve));
- list.addFilter(infoColFamilyList);
- }
- return list;
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- byte[] rowKey =
- FlowRunRowKey.getRowKey(clusterId, userId, flowName, flowRunId);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- get.setFilter(filterList);
- }
- return table.getResult(hbaseConf, conn, get);
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException {
- Scan scan = new Scan();
- scan.setRowPrefixFilter(
- FlowRunRowKey.getRowKeyPrefix(clusterId, userId, flowName));
- FilterList newList = new FilterList();
- newList.addFilter(new PageFilter(limit));
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- newList.addFilter(filterList);
- }
- scan.setFilter(newList);
- return table.getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- FlowRunEntity flowRun = new FlowRunEntity();
- flowRun.setUser(userId);
- flowRun.setName(flowName);
- if (singleEntityRead) {
- flowRun.setRunId(flowRunId);
- } else {
- FlowRunRowKey rowKey = FlowRunRowKey.parseRowKey(result.getRow());
- flowRun.setRunId(rowKey.getFlowRunId());
- }
-
- // read the start time
- Long startTime = (Long)FlowRunColumn.MIN_START_TIME.readResult(result);
- if (startTime != null) {
- flowRun.setStartTime(startTime.longValue());
- }
- if (!singleEntityRead && (flowRun.getStartTime() < createdTimeBegin ||
- flowRun.getStartTime() > createdTimeEnd)) {
- return null;
- }
-
- // read the end time if available
- Long endTime = (Long)FlowRunColumn.MAX_END_TIME.readResult(result);
- if (endTime != null) {
- flowRun.setMaxEndTime(endTime.longValue());
- }
-
- // read the flow version
- String version = (String)FlowRunColumn.FLOW_VERSION.readResult(result);
- if (version != null) {
- flowRun.setVersion(version);
- }
-
- // read metrics
- if (singleEntityRead || fieldsToRetrieve.contains(Field.METRICS)) {
- readMetrics(flowRun, result, FlowRunColumnPrefix.METRIC);
- }
-
- // set the id
- flowRun.setId(flowRun.getId());
- return flowRun;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
deleted file mode 100644
index 784dfd5..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/GenericEntityReader.java
+++ /dev/null
@@ -1,496 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Get;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.client.Scan;
-import org.apache.hadoop.hbase.filter.BinaryComparator;
-import org.apache.hadoop.hbase.filter.BinaryPrefixComparator;
-import org.apache.hadoop.hbase.filter.FamilyFilter;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.hbase.filter.QualifierFilter;
-import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FilterList.Operator;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEvent;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.application.ApplicationTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.apptoflow.AppToFlowTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.Separator;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.TimelineStorageUtils;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnFamily;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
-import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
-
-import com.google.common.base.Preconditions;
-
-/**
- * Timeline entity reader for generic entities that are stored in the entity
- * table.
- */
-class GenericEntityReader extends TimelineEntityReader {
- private static final EntityTable ENTITY_TABLE = new EntityTable();
- private static final Log LOG = LogFactory.getLog(GenericEntityReader.class);
-
- /**
- * Used to look up the flow context.
- */
- private final AppToFlowTable appToFlowTable = new AppToFlowTable();
-
- public GenericEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, limit,
- createdTimeBegin, createdTimeEnd, modifiedTimeBegin, modifiedTimeEnd,
- relatesTo, isRelatedTo, infoFilters, configFilters, metricFilters,
- eventFilters, confsToRetrieve, metricsToRetrieve, fieldsToRetrieve,
- sortedKeys);
- }
-
- public GenericEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- super(userId, clusterId, flowName, flowRunId, appId, entityType, entityId,
- confsToRetrieve, metricsToRetrieve, fieldsToRetrieve);
- }
-
- /**
- * Uses the {@link EntityTable}.
- */
- protected BaseTable<?> getTable() {
- return ENTITY_TABLE;
- }
-
- @Override
- protected FilterList constructFilterListBasedOnFields() {
- FilterList list = new FilterList(Operator.MUST_PASS_ONE);
- // Fetch all the columns.
- if (fieldsToRetrieve.contains(Field.ALL) &&
- (confsToRetrieve == null ||
- confsToRetrieve.getFilterList().isEmpty()) &&
- (metricsToRetrieve == null ||
- metricsToRetrieve.getFilterList().isEmpty())) {
- return list;
- }
- FilterList infoColFamilyList = new FilterList();
- // By default fetch everything in INFO column family.
- FamilyFilter infoColumnFamily =
- new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(EntityColumnFamily.INFO.getBytes()));
- infoColFamilyList.addFilter(infoColumnFamily);
- // Events not required.
- if (!fieldsToRetrieve.contains(Field.EVENTS) &&
- !fieldsToRetrieve.contains(Field.ALL) && eventFilters == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.EVENT.getColumnPrefixBytes(""))));
- }
- // info not required.
- if (!fieldsToRetrieve.contains(Field.INFO) &&
- !fieldsToRetrieve.contains(Field.ALL) && infoFilters == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.INFO.getColumnPrefixBytes(""))));
- }
- // is related to not required.
- if (!fieldsToRetrieve.contains(Field.IS_RELATED_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && isRelatedTo == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.IS_RELATED_TO.getColumnPrefixBytes(""))));
- }
- // relates to not required.
- if (!fieldsToRetrieve.contains(Field.RELATES_TO) &&
- !fieldsToRetrieve.contains(Field.ALL) && relatesTo == null) {
- infoColFamilyList.addFilter(
- new QualifierFilter(CompareOp.NOT_EQUAL,
- new BinaryPrefixComparator(
- EntityColumnPrefix.RELATES_TO.getColumnPrefixBytes(""))));
- }
- list.addFilter(infoColFamilyList);
- if ((fieldsToRetrieve.contains(Field.CONFIGS) || configFilters != null) ||
- (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty())) {
- FilterList filterCfg =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(EntityColumnFamily.CONFIGS.getBytes())));
- if (confsToRetrieve != null &&
- !confsToRetrieve.getFilterList().isEmpty()) {
- filterCfg.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.CONFIG, confsToRetrieve));
- }
- list.addFilter(filterCfg);
- }
- if ((fieldsToRetrieve.contains(Field.METRICS) || metricFilters != null) ||
- (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty())) {
- FilterList filterMetrics =
- new FilterList(new FamilyFilter(CompareOp.EQUAL,
- new BinaryComparator(EntityColumnFamily.METRICS.getBytes())));
- if (metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- filterMetrics.addFilter(TimelineFilterUtils.createHBaseFilterList(
- EntityColumnPrefix.METRIC, metricsToRetrieve));
- }
- list.addFilter(filterMetrics);
- }
- return list;
- }
-
- protected FlowContext lookupFlowContext(String clusterId, String appId,
- Configuration hbaseConf, Connection conn) throws IOException {
- byte[] rowKey = AppToFlowRowKey.getRowKey(clusterId, appId);
- Get get = new Get(rowKey);
- Result result = appToFlowTable.getResult(hbaseConf, conn, get);
- if (result != null && !result.isEmpty()) {
- return new FlowContext(
- AppToFlowColumn.USER_ID.readResult(result).toString(),
- AppToFlowColumn.FLOW_ID.readResult(result).toString(),
- ((Number)AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
- } else {
- throw new IOException(
- "Unable to find the context flow ID and flow run ID for clusterId=" +
- clusterId + ", appId=" + appId);
- }
- }
-
- protected static class FlowContext {
- protected final String userId;
- protected final String flowName;
- protected final Long flowRunId;
- public FlowContext(String user, String flowName, Long flowRunId) {
- this.userId = user;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- }
- }
-
- @Override
- protected void validateParams() {
- Preconditions.checkNotNull(clusterId, "clusterId shouldn't be null");
- Preconditions.checkNotNull(appId, "appId shouldn't be null");
- Preconditions.checkNotNull(entityType, "entityType shouldn't be null");
- if (singleEntityRead) {
- Preconditions.checkNotNull(entityId, "entityId shouldn't be null");
- }
- }
-
- @Override
- protected void augmentParams(Configuration hbaseConf, Connection conn)
- throws IOException {
- // In reality all three should be null or neither should be null
- if (flowName == null || flowRunId == null || userId == null) {
- FlowContext context =
- lookupFlowContext(clusterId, appId, hbaseConf, conn);
- flowName = context.flowName;
- flowRunId = context.flowRunId;
- userId = context.userId;
- }
- if (fieldsToRetrieve == null) {
- fieldsToRetrieve = EnumSet.noneOf(Field.class);
- }
- if (!fieldsToRetrieve.contains(Field.CONFIGS) &&
- confsToRetrieve != null && !confsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.CONFIGS);
- }
- if (!fieldsToRetrieve.contains(Field.METRICS) &&
- metricsToRetrieve != null &&
- !metricsToRetrieve.getFilterList().isEmpty()) {
- fieldsToRetrieve.add(Field.METRICS);
- }
- if (!singleEntityRead) {
- if (limit == null || limit < 0) {
- limit = TimelineReader.DEFAULT_LIMIT;
- }
- if (createdTimeBegin == null) {
- createdTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (createdTimeEnd == null) {
- createdTimeEnd = DEFAULT_END_TIME;
- }
- if (modifiedTimeBegin == null) {
- modifiedTimeBegin = DEFAULT_BEGIN_TIME;
- }
- if (modifiedTimeEnd == null) {
- modifiedTimeEnd = DEFAULT_END_TIME;
- }
- }
- }
-
- @Override
- protected Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException {
- byte[] rowKey =
- EntityRowKey.getRowKey(clusterId, userId, flowName, flowRunId, appId,
- entityType, entityId);
- Get get = new Get(rowKey);
- get.setMaxVersions(Integer.MAX_VALUE);
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- get.setFilter(filterList);
- }
- return table.getResult(hbaseConf, conn, get);
- }
-
- @Override
- protected ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException {
- // Scan through part of the table to find the entities belong to one app
- // and one type
- Scan scan = new Scan();
- scan.setRowPrefixFilter(EntityRowKey.getRowKeyPrefix(
- clusterId, userId, flowName, flowRunId, appId, entityType));
- scan.setMaxVersions(Integer.MAX_VALUE);
- if (filterList != null && !filterList.getFilters().isEmpty()) {
- scan.setFilter(filterList);
- }
- return table.getResultScanner(hbaseConf, conn, scan);
- }
-
- @Override
- protected TimelineEntity parseEntity(Result result) throws IOException {
- if (result == null || result.isEmpty()) {
- return null;
- }
- TimelineEntity entity = new TimelineEntity();
- String entityType = EntityColumn.TYPE.readResult(result).toString();
- entity.setType(entityType);
- String entityId = EntityColumn.ID.readResult(result).toString();
- entity.setId(entityId);
-
- // fetch created time
- Number createdTime = (Number)EntityColumn.CREATED_TIME.readResult(result);
- entity.setCreatedTime(createdTime.longValue());
- if (!singleEntityRead && (entity.getCreatedTime() < createdTimeBegin ||
- entity.getCreatedTime() > createdTimeEnd)) {
- return null;
- }
-
- // fetch modified time
- Number modifiedTime = (Number)EntityColumn.MODIFIED_TIME.readResult(result);
- entity.setModifiedTime(modifiedTime.longValue());
- if (!singleEntityRead && (entity.getModifiedTime() < modifiedTimeBegin ||
- entity.getModifiedTime() > modifiedTimeEnd)) {
- return null;
- }
-
- // fetch is related to entities
- boolean checkIsRelatedTo = isRelatedTo != null && isRelatedTo.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.IS_RELATED_TO) || checkIsRelatedTo) {
- readRelationship(entity, result, EntityColumnPrefix.IS_RELATED_TO, true);
- if (checkIsRelatedTo && !TimelineStorageUtils.matchRelations(
- entity.getIsRelatedToEntities(), isRelatedTo)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.IS_RELATED_TO)) {
- entity.getIsRelatedToEntities().clear();
- }
- }
-
- // fetch relates to entities
- boolean checkRelatesTo = relatesTo != null && relatesTo.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.RELATES_TO) || checkRelatesTo) {
- readRelationship(entity, result, EntityColumnPrefix.RELATES_TO, false);
- if (checkRelatesTo && !TimelineStorageUtils.matchRelations(
- entity.getRelatesToEntities(), relatesTo)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.RELATES_TO)) {
- entity.getRelatesToEntities().clear();
- }
- }
-
- // fetch info
- boolean checkInfo = infoFilters != null && infoFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.INFO) || checkInfo) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.INFO, false);
- if (checkInfo &&
- !TimelineStorageUtils.matchFilters(entity.getInfo(), infoFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.INFO)) {
- entity.getInfo().clear();
- }
- }
-
- // fetch configs
- boolean checkConfigs = configFilters != null && configFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.CONFIGS) || checkConfigs) {
- readKeyValuePairs(entity, result, EntityColumnPrefix.CONFIG, true);
- if (checkConfigs && !TimelineStorageUtils.matchFilters(
- entity.getConfigs(), configFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.CONFIGS)) {
- entity.getConfigs().clear();
- }
- }
-
- // fetch events
- boolean checkEvents = eventFilters != null && eventFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.EVENTS) || checkEvents) {
- readEvents(entity, result, false);
- if (checkEvents && !TimelineStorageUtils.matchEventFilters(
- entity.getEvents(), eventFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.EVENTS)) {
- entity.getEvents().clear();
- }
- }
-
- // fetch metrics
- boolean checkMetrics = metricFilters != null && metricFilters.size() > 0;
- if (fieldsToRetrieve.contains(Field.ALL) ||
- fieldsToRetrieve.contains(Field.METRICS) || checkMetrics) {
- readMetrics(entity, result, EntityColumnPrefix.METRIC);
- if (checkMetrics && !TimelineStorageUtils.matchMetricFilters(
- entity.getMetrics(), metricFilters)) {
- return null;
- }
- if (!fieldsToRetrieve.contains(Field.ALL) &&
- !fieldsToRetrieve.contains(Field.METRICS)) {
- entity.getMetrics().clear();
- }
- }
- return entity;
- }
-
- /**
- * Helper method for reading relationship.
- */
- protected <T> void readRelationship(
- TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
- boolean isRelatedTo) throws IOException {
- // isRelatedTo and relatesTo are of type Map<String, Set<String>>
- Map<String, Object> columns = prefix.readResults(result);
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- for (String id : Separator.VALUES.splitEncoded(
- column.getValue().toString())) {
- if (isRelatedTo) {
- entity.addIsRelatedToEntity(column.getKey(), id);
- } else {
- entity.addRelatesToEntity(column.getKey(), id);
- }
- }
- }
- }
-
- /**
- * Helper method for reading key-value pairs for either info or config.
- */
- protected <T> void readKeyValuePairs(
- TimelineEntity entity, Result result, ColumnPrefix<T> prefix,
- boolean isConfig) throws IOException {
- // info and configuration are of type Map<String, Object or String>
- Map<String, Object> columns = prefix.readResults(result);
- if (isConfig) {
- for (Map.Entry<String, Object> column : columns.entrySet()) {
- entity.addConfig(column.getKey(), column.getValue().toString());
- }
- } else {
- entity.addInfo(columns);
- }
- }
-
- /**
- * Read events from the entity table or the application table. The column name
- * is of the form "eventId=timestamp=infoKey" where "infoKey" may be omitted
- * if there is no info associated with the event.
- *
- * See {@link EntityTable} and {@link ApplicationTable} for a more detailed
- * schema description.
- */
- protected void readEvents(TimelineEntity entity, Result result,
- boolean isApplication) throws IOException {
- Map<String, TimelineEvent> eventsMap = new HashMap<>();
- Map<?, Object> eventsResult = isApplication ?
- ApplicationColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result) :
- EntityColumnPrefix.EVENT.
- readResultsHavingCompoundColumnQualifiers(result);
- for (Map.Entry<?, Object> eventResult : eventsResult.entrySet()) {
- byte[][] karr = (byte[][])eventResult.getKey();
- // the column name is of the form "eventId=timestamp=infoKey"
- if (karr.length == 3) {
- String id = Bytes.toString(karr[0]);
- long ts = TimelineStorageUtils.invertLong(Bytes.toLong(karr[1]));
- String key = Separator.VALUES.joinEncoded(id, Long.toString(ts));
- TimelineEvent event = eventsMap.get(key);
- if (event == null) {
- event = new TimelineEvent();
- event.setId(id);
- event.setTimestamp(ts);
- eventsMap.put(key, event);
- }
- // handle empty info
- String infoKey = karr[2].length == 0 ? null : Bytes.toString(karr[2]);
- if (infoKey != null) {
- event.addInfo(infoKey, eventResult.getValue());
- }
- } else {
- LOG.warn("incorrectly formatted column name: it will be discarded");
- continue;
- }
- }
- Set<TimelineEvent> eventsSet = new HashSet<>(eventsMap.values());
- entity.addEvents(eventsSet);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
index 96c5a19..bc48cbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/HBaseTimelineReaderImpl.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
+import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader;
+import org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReaderFactory;
public class HBaseTimelineReaderImpl
extends AbstractService implements TimelineReader {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
deleted file mode 100644
index a26c0c2..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReader.java
+++ /dev/null
@@ -1,274 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.io.IOException;
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.NavigableSet;
-import java.util.Set;
-import java.util.TreeSet;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.Result;
-import org.apache.hadoop.hbase.client.ResultScanner;
-import org.apache.hadoop.hbase.filter.FilterList;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTable;
-import org.apache.hadoop.yarn.server.timelineservice.storage.common.ColumnPrefix;
-
-/**
- * The base class for reading and deserializing timeline entities from the
- * HBase storage. Different types can be defined for different types of the
- * entities that are being requested.
- */
-abstract class TimelineEntityReader {
- private static final Log LOG = LogFactory.getLog(TimelineEntityReader.class);
- protected static final long DEFAULT_BEGIN_TIME = 0L;
- protected static final long DEFAULT_END_TIME = Long.MAX_VALUE;
-
- protected final boolean singleEntityRead;
-
- protected String userId;
- protected String clusterId;
- protected String flowName;
- protected Long flowRunId;
- protected String appId;
- protected String entityType;
- protected EnumSet<Field> fieldsToRetrieve;
- // used only for a single entity read mode
- protected String entityId;
- // used only for multiple entity read mode
- protected Long limit;
- protected Long createdTimeBegin;
- protected Long createdTimeEnd;
- protected Long modifiedTimeBegin;
- protected Long modifiedTimeEnd;
- protected Map<String, Set<String>> relatesTo;
- protected Map<String, Set<String>> isRelatedTo;
- protected Map<String, Object> infoFilters;
- protected Map<String, String> configFilters;
- protected Set<String> metricFilters;
- protected Set<String> eventFilters;
- protected TimelineFilterList confsToRetrieve;
- protected TimelineFilterList metricsToRetrieve;
-
- /**
- * Main table the entity reader uses.
- */
- protected BaseTable<?> table;
-
- /**
- * Specifies whether keys for this table are sorted in a manner where entities
- * can be retrieved by created time. If true, it will be sufficient to collect
- * the first results as specified by the limit. Otherwise all matched entities
- * will be fetched and then limit applied.
- */
- private boolean sortedKeys = false;
-
- /**
- * Instantiates a reader for multiple-entity reads.
- */
- protected TimelineEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confsToRetrieve, TimelineFilterList metricsToRetrieve,
- EnumSet<Field> fieldsToRetrieve, boolean sortedKeys) {
- this.singleEntityRead = false;
- this.sortedKeys = sortedKeys;
- this.userId = userId;
- this.clusterId = clusterId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- this.appId = appId;
- this.entityType = entityType;
- this.fieldsToRetrieve = fieldsToRetrieve;
- this.limit = limit;
- this.createdTimeBegin = createdTimeBegin;
- this.createdTimeEnd = createdTimeEnd;
- this.modifiedTimeBegin = modifiedTimeBegin;
- this.modifiedTimeEnd = modifiedTimeEnd;
- this.relatesTo = relatesTo;
- this.isRelatedTo = isRelatedTo;
- this.infoFilters = infoFilters;
- this.configFilters = configFilters;
- this.metricFilters = metricFilters;
- this.eventFilters = eventFilters;
- this.confsToRetrieve = confsToRetrieve;
- this.metricsToRetrieve = metricsToRetrieve;
-
- this.table = getTable();
- }
-
- /**
- * Instantiates a reader for single-entity reads.
- */
- protected TimelineEntityReader(String userId, String clusterId,
- String flowName, Long flowRunId, String appId, String entityType,
- String entityId, TimelineFilterList confsToRetrieve,
- TimelineFilterList metricsToRetrieve, EnumSet<Field> fieldsToRetrieve) {
- this.singleEntityRead = true;
- this.userId = userId;
- this.clusterId = clusterId;
- this.flowName = flowName;
- this.flowRunId = flowRunId;
- this.appId = appId;
- this.entityType = entityType;
- this.fieldsToRetrieve = fieldsToRetrieve;
- this.entityId = entityId;
- this.confsToRetrieve = confsToRetrieve;
- this.metricsToRetrieve = metricsToRetrieve;
-
- this.table = getTable();
- }
-
- /**
- * Creates a {@link FilterList} based on fields, confs and metrics to
- * retrieve. This filter list will be set in Scan/Get objects to trim down
- * results fetched from HBase back-end storage.
- * @return a {@link FilterList} object.
- */
- protected abstract FilterList constructFilterListBasedOnFields();
-
- /**
- * Reads and deserializes a single timeline entity from the HBase storage.
- */
- public TimelineEntity readEntity(Configuration hbaseConf, Connection conn)
- throws IOException {
- validateParams();
- augmentParams(hbaseConf, conn);
-
- FilterList filterList = constructFilterListBasedOnFields();
- Result result = getResult(hbaseConf, conn, filterList);
- if (result == null || result.isEmpty()) {
- // Could not find a matching row.
- LOG.info("Cannot find matching entity of type " + entityType);
- return null;
- }
- return parseEntity(result);
- }
-
- /**
- * Reads and deserializes a set of timeline entities from the HBase storage.
- * It goes through all the results available, and returns the number of
- * entries as specified in the limit in the entity's natural sort order.
- */
- public Set<TimelineEntity> readEntities(Configuration hbaseConf,
- Connection conn) throws IOException {
- validateParams();
- augmentParams(hbaseConf, conn);
-
- NavigableSet<TimelineEntity> entities = new TreeSet<>();
- FilterList filterList = constructFilterListBasedOnFields();
- ResultScanner results = getResults(hbaseConf, conn, filterList);
- try {
- for (Result result : results) {
- TimelineEntity entity = parseEntity(result);
- if (entity == null) {
- continue;
- }
- entities.add(entity);
- if (!sortedKeys) {
- if (entities.size() > limit) {
- entities.pollLast();
- }
- } else {
- if (entities.size() == limit) {
- break;
- }
- }
- }
- return entities;
- } finally {
- results.close();
- }
- }
-
- /**
- * Returns the main table to be used by the entity reader.
- */
- protected abstract BaseTable<?> getTable();
-
- /**
- * Validates the required parameters to read the entities.
- */
- protected abstract void validateParams();
-
- /**
- * Sets certain parameters to defaults if the values are not provided.
- */
- protected abstract void augmentParams(Configuration hbaseConf,
- Connection conn) throws IOException;
-
- /**
- * Fetches a {@link Result} instance for a single-entity read.
- *
- * @return the {@link Result} instance or null if no such record is found.
- */
- protected abstract Result getResult(Configuration hbaseConf, Connection conn,
- FilterList filterList) throws IOException;
-
- /**
- * Fetches a {@link ResultScanner} for a multi-entity read.
- */
- protected abstract ResultScanner getResults(Configuration hbaseConf,
- Connection conn, FilterList filterList) throws IOException;
-
- /**
- * Given a {@link Result} instance, deserializes and creates a
- * {@link TimelineEntity}.
- *
- * @return the {@link TimelineEntity} instance, or null if the {@link Result}
- * is null or empty.
- */
- protected abstract TimelineEntity parseEntity(Result result)
- throws IOException;
-
- /**
- * Helper method for reading and deserializing {@link TimelineMetric} objects
- * using the specified column prefix. The timeline metrics then are added to
- * the given timeline entity.
- */
- protected void readMetrics(TimelineEntity entity, Result result,
- ColumnPrefix<?> columnPrefix) throws IOException {
- NavigableMap<String, NavigableMap<Long, Number>> metricsResult =
- columnPrefix.readResultsWithTimestamps(result);
- for (Map.Entry<String, NavigableMap<Long, Number>> metricResult:
- metricsResult.entrySet()) {
- TimelineMetric metric = new TimelineMetric();
- metric.setId(metricResult.getKey());
- // Simply assume that if the value set contains more than 1 elements, the
- // metric is a TIME_SERIES metric, otherwise, it's a SINGLE_VALUE metric
- metric.setType(metricResult.getValue().size() > 1 ?
- TimelineMetric.Type.TIME_SERIES : TimelineMetric.Type.SINGLE_VALUE);
- metric.addValues(metricResult.getValue());
- entity.addMetric(metric);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/61737325/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
deleted file mode 100644
index 36ed4ca..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineEntityReaderFactory.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.yarn.server.timelineservice.storage;
-
-import java.util.EnumSet;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntityType;
-import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
-import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
-
-/**
- * Factory methods for instantiating a timeline entity reader.
- */
-class TimelineEntityReaderFactory {
- /**
- * Creates a timeline entity reader instance for reading a single entity with
- * the specified input.
- */
- public static TimelineEntityReader createSingleEntityReader(String userId,
- String clusterId, String flowName, Long flowRunId, String appId,
- String entityType, String entityId, TimelineFilterList confs,
- TimelineFilterList metrics, EnumSet<Field> fieldsToRetrieve) {
- // currently the types that are handled separate from the generic entity
- // table are application, flow run, and flow activity entities
- if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
- return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
- return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
- return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, fieldsToRetrieve);
- } else {
- // assume we're dealing with a generic entity read
- return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, entityId, confs, metrics, fieldsToRetrieve);
- }
- }
-
- /**
- * Creates a timeline entity reader instance for reading set of entities with
- * the specified input and predicates.
- */
- public static TimelineEntityReader createMultipleEntitiesReader(String userId,
- String clusterId, String flowName, Long flowRunId, String appId,
- String entityType, Long limit, Long createdTimeBegin, Long createdTimeEnd,
- Long modifiedTimeBegin, Long modifiedTimeEnd,
- Map<String, Set<String>> relatesTo, Map<String, Set<String>> isRelatedTo,
- Map<String, Object> infoFilters, Map<String, String> configFilters,
- Set<String> metricFilters, Set<String> eventFilters,
- TimelineFilterList confs, TimelineFilterList metrics,
- EnumSet<Field> fieldsToRetrieve) {
- // currently the types that are handled separate from the generic entity
- // table are application, flow run, and flow activity entities
- if (TimelineEntityType.YARN_APPLICATION.matches(entityType)) {
- return new ApplicationEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_ACTIVITY.matches(entityType)) {
- return new FlowActivityEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters,
- fieldsToRetrieve);
- } else if (TimelineEntityType.YARN_FLOW_RUN.matches(entityType)) {
- return new FlowRunEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve);
- } else {
- // assume we're dealing with a generic entity read
- return new GenericEntityReader(userId, clusterId, flowName, flowRunId,
- appId, entityType, limit, createdTimeBegin, createdTimeEnd,
- modifiedTimeBegin, modifiedTimeEnd, relatesTo, isRelatedTo,
- infoFilters, configFilters, metricFilters, eventFilters, confs,
- metrics, fieldsToRetrieve, false);
- }
- }
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-4238. createdTime and
modifiedTime is not reported while publishing entities to ATSv2. (Varun
Saxena via Naganarasimha G R)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
index e864d61..b7804e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineReaderImpl.java
@@ -107,7 +107,6 @@ public class TestFileSystemTimelineReaderImpl {
entity11.setId("id_1");
entity11.setType("app");
entity11.setCreatedTime(1425016502000L);
- entity11.setModifiedTime(1425016502050L);
Map<String, Object> info1 = new HashMap<String, Object>();
info1.put("info1", "val1");
entity11.addInfo(info1);
@@ -136,7 +135,6 @@ public class TestFileSystemTimelineReaderImpl {
TimelineEntity entity12 = new TimelineEntity();
entity12.setId("id_1");
entity12.setType("app");
- entity12.setModifiedTime(1425016503000L);
configs.clear();
configs.put("config_2", "23");
configs.put("config_3", "abc");
@@ -166,7 +164,6 @@ public class TestFileSystemTimelineReaderImpl {
entity2.setId("id_2");
entity2.setType("app");
entity2.setCreatedTime(1425016501050L);
- entity2.setModifiedTime(1425016502010L);
Map<String, Object> info2 = new HashMap<String, Object>();
info1.put("info2", 4);
entity2.addInfo(info2);
@@ -203,7 +200,6 @@ public class TestFileSystemTimelineReaderImpl {
entity3.setId("id_3");
entity3.setType("app");
entity3.setCreatedTime(1425016501050L);
- entity3.setModifiedTime(1425016502010L);
Map<String, Object> info3 = new HashMap<String, Object>();
info3.put("info2", 3.5);
entity3.addInfo(info3);
@@ -239,7 +235,6 @@ public class TestFileSystemTimelineReaderImpl {
entity4.setId("id_4");
entity4.setType("app");
entity4.setCreatedTime(1425016502050L);
- entity4.setModifiedTime(1425016503010L);
TimelineEvent event44 = new TimelineEvent();
event44.setId("event_4");
event44.setTimestamp(1425016502003L);
@@ -252,7 +247,6 @@ public class TestFileSystemTimelineReaderImpl {
entity5.setId("id_5");
entity5.setType("app");
entity5.setCreatedTime(1425016502050L);
- entity5.setModifiedTime(1425016503010L);
writeEntityFile(entity5, appDir2);
}
@@ -263,7 +257,7 @@ public class TestFileSystemTimelineReaderImpl {
@Test
public void testGetEntityDefaultView() throws Exception {
// If no fields are specified, entity is returned with default view i.e.
- // only the id, created and modified time
+ // only the id, type and created time.
TimelineEntity result =
reader.getEntity("user1", "cluster1", "flow1", 1L, "app1",
"app", "id_1", null, null, null);
@@ -271,7 +265,6 @@ public class TestFileSystemTimelineReaderImpl {
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime());
- Assert.assertEquals(1425016503000L, result.getModifiedTime());
Assert.assertEquals(0, result.getConfigs().size());
Assert.assertEquals(0, result.getMetrics().size());
}
@@ -286,7 +279,6 @@ public class TestFileSystemTimelineReaderImpl {
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime());
- Assert.assertEquals(1425016503000L, result.getModifiedTime());
Assert.assertEquals(0, result.getConfigs().size());
Assert.assertEquals(0, result.getMetrics().size());
}
@@ -303,7 +295,6 @@ public class TestFileSystemTimelineReaderImpl {
(new TimelineEntity.Identifier("app", "id_5")).toString(),
result.getIdentifier().toString());
Assert.assertEquals(1425016502050L, result.getCreatedTime());
- Assert.assertEquals(1425016503010L, result.getModifiedTime());
}
@Test
@@ -317,7 +308,6 @@ public class TestFileSystemTimelineReaderImpl {
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime());
- Assert.assertEquals(1425016503000L, result.getModifiedTime());
Assert.assertEquals(3, result.getConfigs().size());
Assert.assertEquals(3, result.getMetrics().size());
Assert.assertEquals(1, result.getInfo().size());
@@ -335,7 +325,6 @@ public class TestFileSystemTimelineReaderImpl {
(new TimelineEntity.Identifier("app", "id_1")).toString(),
result.getIdentifier().toString());
Assert.assertEquals(1425016502000L, result.getCreatedTime());
- Assert.assertEquals(1425016503000L, result.getModifiedTime());
Assert.assertEquals(3, result.getConfigs().size());
Assert.assertEquals(3, result.getMetrics().size());
// All fields including events will be returned.
@@ -346,8 +335,8 @@ public class TestFileSystemTimelineReaderImpl {
public void testGetAllEntities() throws Exception {
Set<TimelineEntity> result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, null, null, null,
- null, null, null, null);
+ null, null, null, null, null, null, null, null, null, null, null,
+ null);
// All 3 entities will be returned
Assert.assertEquals(4, result.size());
}
@@ -356,8 +345,8 @@ public class TestFileSystemTimelineReaderImpl {
public void testGetEntitiesWithLimit() throws Exception {
Set<TimelineEntity> result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- 2L, null, null, null, null, null, null, null, null, null,
- null, null, null, null);
+ 2L, null, null, null, null, null, null, null, null, null, null,
+ null);
Assert.assertEquals(2, result.size());
// Needs to be rewritten once hashcode and equals for
// TimelineEntity is implemented
@@ -370,8 +359,8 @@ public class TestFileSystemTimelineReaderImpl {
}
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- 3L, null, null, null, null, null, null, null, null, null,
- null, null, null, null);
+ 3L, null, null, null, null, null, null, null, null, null, null,
+ null);
// Even though 2 entities out of 4 have same created time, one entity
// is left out due to limit
Assert.assertEquals(3, result.size());
@@ -383,7 +372,7 @@ public class TestFileSystemTimelineReaderImpl {
Set<TimelineEntity> result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
null, 1425016502030L, 1425016502060L, null, null, null, null, null,
- null, null, null, null, null, null);
+ null, null, null, null);
Assert.assertEquals(1, result.size());
// Only one entity with ID id_4 should be returned.
for (TimelineEntity entity : result) {
@@ -396,7 +385,7 @@ public class TestFileSystemTimelineReaderImpl {
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
null, null, 1425016502010L, null, null, null, null, null, null,
- null, null, null, null, null);
+ null, null, null);
Assert.assertEquals(3, result.size());
for (TimelineEntity entity : result) {
if (entity.getId().equals("id_4")) {
@@ -408,50 +397,13 @@ public class TestFileSystemTimelineReaderImpl {
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
null, 1425016502010L, null, null, null, null, null, null, null,
- null, null, null, null, null);
+ null, null, null);
Assert.assertEquals(1, result.size());
for (TimelineEntity entity : result) {
if (!entity.getId().equals("id_4")) {
Assert.fail("Incorrect filtering based on created time range");
}
}
-
- // Get entities based on modified time start and end time range.
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, 1425016502090L, 1425016503020L, null, null, null,
- null, null, null, null, null, null);
- Assert.assertEquals(2, result.size());
- // Two entities with IDs' id_1 and id_4 should be returned.
- for (TimelineEntity entity : result) {
- if (!entity.getId().equals("id_1") && !entity.getId().equals("id_4")) {
- Assert.fail("Incorrect filtering based on modified time range");
- }
- }
-
- // Get entities if only modified time end is specified.
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, 1425016502090L, null, null, null, null,
- null, null, null, null, null);
- Assert.assertEquals(2, result.size());
- for (TimelineEntity entity : result) {
- if (entity.getId().equals("id_1") || entity.getId().equals("id_4")) {
- Assert.fail("Incorrect filtering based on modified time range");
- }
- }
-
- // Get entities if only modified time start is specified.
- result =
- reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, 1425016503005L, null, null, null, null, null,
- null, null, null, null, null);
- Assert.assertEquals(1, result.size());
- for (TimelineEntity entity : result) {
- if (!entity.getId().equals("id_4")) {
- Assert.fail("Incorrect filtering based on modified time range");
- }
- }
}
@Test
@@ -461,8 +413,8 @@ public class TestFileSystemTimelineReaderImpl {
infoFilters.put("info2", 3.5);
Set<TimelineEntity> result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, infoFilters, null, null,
- null, null, null, null);
+ null, null, null, null, null, infoFilters, null, null, null, null,
+ null, null);
Assert.assertEquals(1, result.size());
// Only one entity with ID id_3 should be returned.
for (TimelineEntity entity : result) {
@@ -477,8 +429,8 @@ public class TestFileSystemTimelineReaderImpl {
configFilters.put("config_3", "abc");
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, null, configFilters, null,
- null, null, null, null);
+ null, null, null, null, null, null, configFilters, null, null, null,
+ null, null);
Assert.assertEquals(2, result.size());
for (TimelineEntity entity : result) {
if (!entity.getId().equals("id_1") && !entity.getId().equals("id_3")) {
@@ -492,8 +444,8 @@ public class TestFileSystemTimelineReaderImpl {
eventFilters.add("event_4");
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, null, null, null,
- eventFilters, null, null, null);
+ null, null, null, null, null, null, null, null, eventFilters, null,
+ null, null);
Assert.assertEquals(1, result.size());
for (TimelineEntity entity : result) {
if (!entity.getId().equals("id_3")) {
@@ -506,8 +458,8 @@ public class TestFileSystemTimelineReaderImpl {
metricFilters.add("metric3");
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, null, null, null, metricFilters,
- null, null, null, null);
+ null, null, null, null, null, null, null, metricFilters, null, null,
+ null, null);
Assert.assertEquals(2, result.size());
// Two entities with IDs' id_1 and id_2 should be returned.
for (TimelineEntity entity : result) {
@@ -526,8 +478,8 @@ public class TestFileSystemTimelineReaderImpl {
relatesTo.put("flow", relatesToIds);
Set<TimelineEntity> result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, relatesTo, null, null, null, null,
- null, null, null, null);
+ null, null, null, relatesTo, null, null, null, null, null, null,
+ null, null);
Assert.assertEquals(1, result.size());
// Only one entity with ID id_1 should be returned.
for (TimelineEntity entity : result) {
@@ -543,8 +495,8 @@ public class TestFileSystemTimelineReaderImpl {
isRelatedTo.put("type1", isRelatedToIds);
result =
reader.getEntities("user1", "cluster1", "flow1", 1L, "app1", "app",
- null, null, null, null, null, null, isRelatedTo, null, null, null,
- null, null, null, null);
+ null, null, null, null, isRelatedTo, null, null, null, null, null,
+ null, null);
Assert.assertEquals(2, result.size());
// Two entities with IDs' id_1 and id_3 should be returned.
for (TimelineEntity entity : result) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
index 50a9f60..5ce7d3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestFileSystemTimelineWriterImpl.java
@@ -49,7 +49,6 @@ public class TestFileSystemTimelineWriterImpl {
entity.setId(id);
entity.setType(type);
entity.setCreatedTime(1425016501000L);
- entity.setModifiedTime(1425016502000L);
te.addEntity(entity);
FileSystemTimelineWriterImpl fsi = null;
@@ -80,7 +79,7 @@ public class TestFileSystemTimelineWriterImpl {
assertTrue(!(f.exists()));
} finally {
if (fsi != null) {
- fsi.stop();
+ fsi.close();
FileUtils.deleteDirectory(new File(fsi.getOutputRoot()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
index 3198307..b74f5a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestHBaseTimelineStorage.java
@@ -74,7 +74,6 @@ import org.junit.Test;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
-import com.google.common.collect.Maps;
/**
* Various tests to test writing entities to HBase and reading them back from
@@ -112,9 +111,7 @@ public class TestHBaseTimelineStorage {
entity.setId(id);
entity.setType(TimelineEntityType.YARN_APPLICATION.toString());
Long cTime = 1425016501000L;
- Long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap = new HashMap<String, Object>();
infoMap.put("infoMapKey1", "infoMapValue1");
@@ -177,7 +174,6 @@ public class TestHBaseTimelineStorage {
entity1.setId(id1);
entity1.setType(TimelineEntityType.YARN_APPLICATION.toString());
entity1.setCreatedTime(cTime);
- entity1.setModifiedTime(mTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap1 = new HashMap<String, Object>();
@@ -235,7 +231,6 @@ public class TestHBaseTimelineStorage {
entity2.setId(id2);
entity2.setType(TimelineEntityType.YARN_APPLICATION.toString());
entity2.setCreatedTime(cTime);
- entity2.setModifiedTime(mTime);
te2.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
@@ -270,9 +265,7 @@ public class TestHBaseTimelineStorage {
entity.setId(id);
entity.setType(type);
Long cTime = 1425016501000L;
- Long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap = new HashMap<String, Object>();
infoMap.put("infoMapKey1", "infoMapValue1");
@@ -333,7 +326,6 @@ public class TestHBaseTimelineStorage {
entity1.setId(id1);
entity1.setType(type);
entity1.setCreatedTime(cTime);
- entity1.setModifiedTime(mTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap1 = new HashMap<String, Object>();
@@ -390,7 +382,6 @@ public class TestHBaseTimelineStorage {
entity2.setId(id2);
entity2.setType(type);
entity2.setCreatedTime(cTime);
- entity2.setModifiedTime(mTime);
te.addEntity(entity2);
HBaseTimelineWriterImpl hbi = null;
try {
@@ -444,9 +435,7 @@ public class TestHBaseTimelineStorage {
String appId = "application_1000178881110_2002";
entity.setId(appId);
long cTime = 1425016501000L;
- long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap = new HashMap<String, Object>();
@@ -522,7 +511,7 @@ public class TestHBaseTimelineStorage {
Result result = new ApplicationTable().getResult(c1, conn, get);
assertTrue(result != null);
- assertEquals(16, result.size());
+ assertEquals(15, result.size());
// check the row key
byte[] row1 = result.getRow();
@@ -538,10 +527,6 @@ public class TestHBaseTimelineStorage {
long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
- val = (Number) ApplicationColumn.MODIFIED_TIME.readResult(result);
- long mTime1 = val.longValue();
- assertEquals(mTime1, mTime);
-
Map<String, Object> infoColumns =
ApplicationColumnPrefix.INFO.readResults(result);
assertEquals(infoMap, infoColumns);
@@ -599,7 +584,6 @@ public class TestHBaseTimelineStorage {
assertEquals(TimelineEntityType.YARN_APPLICATION.toString(),
e1.getType());
assertEquals(cTime, e1.getCreatedTime());
- assertEquals(mTime, e1.getModifiedTime());
Map<String, Object> infoMap2 = e1.getInfo();
assertEquals(infoMap, infoMap2);
@@ -635,9 +619,7 @@ public class TestHBaseTimelineStorage {
entity.setId(id);
entity.setType(type);
long cTime = 1425016501000L;
- long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
// add the info map in Timeline Entity
Map<String, Object> infoMap = new HashMap<String, Object>();
@@ -736,10 +718,6 @@ public class TestHBaseTimelineStorage {
long cTime1 = val.longValue();
assertEquals(cTime1, cTime);
- val = (Number) EntityColumn.MODIFIED_TIME.readResult(result);
- long mTime1 = val.longValue();
- assertEquals(mTime1, mTime);
-
Map<String, Object> infoColumns =
EntityColumnPrefix.INFO.readResults(result);
assertEquals(infoMap, infoColumns);
@@ -790,7 +768,7 @@ public class TestHBaseTimelineStorage {
}
}
assertEquals(1, rowCount);
- assertEquals(17, colCount);
+ assertEquals(16, colCount);
// read the timeline entity using the reader this time
TimelineEntity e1 = reader.getEntity(user, cluster, flow, runid, appName,
@@ -798,8 +776,7 @@ public class TestHBaseTimelineStorage {
EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = reader.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, null, null,
- EnumSet.of(TimelineReader.Field.ALL));
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
@@ -807,7 +784,6 @@ public class TestHBaseTimelineStorage {
assertEquals(id, e1.getId());
assertEquals(type, e1.getType());
assertEquals(cTime, e1.getCreatedTime());
- assertEquals(mTime, e1.getModifiedTime());
Map<String, Object> infoMap2 = e1.getInfo();
assertEquals(infoMap, infoMap2);
@@ -1041,8 +1017,7 @@ public class TestHBaseTimelineStorage {
EnumSet.of(TimelineReader.Field.ALL));
Set<TimelineEntity> es1 = reader.getEntities(user, cluster, flow, runid,
appName, entity.getType(), null, null, null, null, null, null, null,
- null, null, null, null, null, null,
- EnumSet.of(TimelineReader.Field.ALL));
+ null, null, null, null, EnumSet.of(TimelineReader.Field.ALL));
assertNotNull(e1);
assertEquals(1, es1.size());
@@ -1069,7 +1044,6 @@ public class TestHBaseTimelineStorage {
String appId = "application_1000178881110_2002";
entityApp.setId(appId);
entityApp.setCreatedTime(1425016501000L);
- entityApp.setModifiedTime(1425026901000L);
// add metrics with floating point values
Set<TimelineMetric> metricsApp = new HashSet<>();
TimelineMetric mApp = new TimelineMetric();
@@ -1089,7 +1063,6 @@ public class TestHBaseTimelineStorage {
entity.setId("hello");
entity.setType("world");
entity.setCreatedTime(1425016501000L);
- entity.setModifiedTime(1425026901000L);
// add metrics with floating point values
Set<TimelineMetric> metricsEntity = new HashSet<>();
TimelineMetric mEntity = new TimelineMetric();
@@ -1135,8 +1108,8 @@ public class TestHBaseTimelineStorage {
assertEquals(1, e1.getIsRelatedToEntities().size());
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null, null,
- null, EnumSet.of(Field.ALL));
+ null, null, null, null, null, null, null, null, null, null, null,
+ EnumSet.of(Field.ALL));
assertEquals(3, es1.size());
}
@@ -1151,8 +1124,7 @@ public class TestHBaseTimelineStorage {
e1.getRelatesToEntities().isEmpty());
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null, null,
- null, null);
+ null, null, null, null, null, null, null, null, null, null, null, null);
assertEquals(3, es1.size());
for (TimelineEntity e : es1) {
assertTrue(e.getInfo().isEmpty() && e.getConfigs().isEmpty() &&
@@ -1172,8 +1144,8 @@ public class TestHBaseTimelineStorage {
assertEquals(0, e1.getIsRelatedToEntities().size());
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null, null,
- null, EnumSet.of(Field.IS_RELATED_TO, Field.METRICS));
+ null, null, null, null, null, null, null, null, null, null, null,
+ EnumSet.of(Field.IS_RELATED_TO, Field.METRICS));
assertEquals(3, es1.size());
int metricsCnt = 0;
int isRelatedToCnt = 0;
@@ -1200,8 +1172,7 @@ public class TestHBaseTimelineStorage {
assertEquals(1, e1.getConfigs().size());
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null,
- list, null, null);
+ null, null, null, null, null, null, null, null, null, list, null, null);
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
@@ -1217,8 +1188,8 @@ public class TestHBaseTimelineStorage {
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
Set<TimelineEntity> entities = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, confFilters, null, null,
- list, null, null);
+ null, null, null, null, null, null, confFilters, null, null, list, null,
+ null);
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1239,8 +1210,7 @@ public class TestHBaseTimelineStorage {
assertEquals(1, e1.getMetrics().size());
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, null, null, null,
- list, null);
+ null, null, null, null, null, null, null, null, null, null, list, null);
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
@@ -1256,8 +1226,8 @@ public class TestHBaseTimelineStorage {
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
Set<TimelineEntity> entities = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, "application_1231111111_1111","world",
- null, null, null, null, null, null, null, null, null, metricFilters,
- null, null, list, null);
+ null, null, null, null, null, null, null, metricFilters, null, null,
+ list, null);
assertEquals(1, entities.size());
int metricCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1278,8 +1248,7 @@ public class TestHBaseTimelineStorage {
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, null, null,
- EnumSet.of(Field.ALL));
+ null, null, null, null, null, null, null, EnumSet.of(Field.ALL));
assertEquals(3, es1.size());
}
@@ -1295,7 +1264,7 @@ public class TestHBaseTimelineStorage {
Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
"some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, null, null, null);
+ null, null, null, null, null, null, null, null);
assertEquals(3, es1.size());
for (TimelineEntity e : es1) {
assertTrue(e.getInfo().isEmpty() && e.getConfigs().isEmpty() &&
@@ -1313,10 +1282,10 @@ public class TestHBaseTimelineStorage {
assertNotNull(e1);
assertEquals(3, e1.getConfigs().size());
assertEquals(0, e1.getIsRelatedToEntities().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
+ Set<TimelineEntity> es1 = reader.getEntities(
+ "user1", "cluster1", "some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, null, null,
+ null, null, null, null, null, null, null,
EnumSet.of(Field.IS_RELATED_TO, Field.METRICS));
assertEquals(3, es1.size());
int metricsCnt = 0;
@@ -1342,10 +1311,10 @@ public class TestHBaseTimelineStorage {
TimelineEntityType.YARN_APPLICATION.toString(), null, list, null, null);
assertNotNull(e1);
assertEquals(1, e1.getConfigs().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
+ Set<TimelineEntity> es1 = reader.getEntities(
+ "user1", "cluster1", "some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, list, null, null);
+ null, null, null, null, null, list, null, null);
int cfgCnt = 0;
for (TimelineEntity entity : es1) {
cfgCnt += entity.getConfigs().size();
@@ -1359,10 +1328,10 @@ public class TestHBaseTimelineStorage {
TimelineFilterList list =
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "cfg_"));
- Set<TimelineEntity> entities = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
+ Set<TimelineEntity> entities = reader.getEntities(
+ "user1", "cluster1", "some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, confFilters, null, null, list, null, null);
+ null, null, confFilters, null, null, list, null, null);
assertEquals(1, entities.size());
int cfgCnt = 0;
for (TimelineEntity entity : entities) {
@@ -1381,10 +1350,10 @@ public class TestHBaseTimelineStorage {
TimelineEntityType.YARN_APPLICATION.toString(), null, null, list, null);
assertNotNull(e1);
assertEquals(1, e1.getMetrics().size());
- Set<TimelineEntity> es1 = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
+ Set<TimelineEntity> es1 = reader.getEntities(
+ "user1", "cluster1", "some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, null, null, null, list, null);
+ null, null, null, null, null, null, list, null);
int metricCnt = 0;
for (TimelineEntity entity : es1) {
metricCnt += entity.getMetrics().size();
@@ -1398,10 +1367,10 @@ public class TestHBaseTimelineStorage {
new TimelineFilterList(Operator.OR,
new TimelinePrefixFilter(TimelineCompareOp.EQUAL, "MAP1_"));
Set<String> metricFilters = ImmutableSet.of("MAP1_SLOT_MILLIS");
- Set<TimelineEntity> entities = reader.getEntities("user1", "cluster1",
- "some_flow_name", 1002345678919L, null,
+ Set<TimelineEntity> entities = reader.getEntities(
+ "user1", "cluster1", "some_flow_name", 1002345678919L, null,
TimelineEntityType.YARN_APPLICATION.toString(), null, null, null, null,
- null, null, null, null, null, metricFilters, null, null, list, null);
+ null, null, null, metricFilters, null, null, list, null);
int metricCnt = 0;
assertEquals(1, entities.size());
for (TimelineEntity entity : entities) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
index de66a17..330adb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
@@ -111,7 +111,6 @@ public class TestPhoenixOfflineAggregationWriterImpl extends BaseTest {
entity.setId(id);
entity.setType(type);
entity.setCreatedTime(1425016501000L);
- entity.setModifiedTime(1425016502000L);
TimelineMetric metric = new TimelineMetric();
metric.setId("HDFS_BYTES_READ");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
index d18613a..a4c06f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestFlowDataGenerator.java
@@ -106,7 +106,6 @@ class TestFlowDataGenerator {
long cTime = 20000000000000L;
long mTime = 1425026901000L;
entity.setCreatedTime(cTime);
- entity.setModifiedTime(mTime);
// add metrics
Set<TimelineMetric> metrics = new HashSet<>();
TimelineMetric m1 = new TimelineMetric();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
index eda1e21..cb3f7ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowActivity.java
@@ -182,7 +182,7 @@ public class TestHBaseStorageFlowActivity {
Set<TimelineEntity> entities =
hbr.getEntities(null, cluster, null, null, null,
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
- null, null, null, null, null, null, null, null, null, null, null);
+ null, null, null, null, null, null, null, null, null);
assertEquals(1, entities.size());
for (TimelineEntity e : entities) {
FlowActivityEntity flowActivity = (FlowActivityEntity)e;
@@ -238,7 +238,7 @@ public class TestHBaseStorageFlowActivity {
Set<TimelineEntity> entities =
hbr.getEntities(user, cluster, flow, null, null,
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
- null, null, null, null, null, null, null, null, null, null, null);
+ null, null, null, null, null, null, null, null, null);
assertEquals(1, entities.size());
for (TimelineEntity e : entities) {
FlowActivityEntity entity = (FlowActivityEntity)e;
@@ -353,7 +353,7 @@ public class TestHBaseStorageFlowActivity {
Set<TimelineEntity> entities =
hbr.getEntities(null, cluster, null, null, null,
TimelineEntityType.YARN_FLOW_ACTIVITY.toString(), 10L, null, null,
- null, null, null, null, null, null, null, null, null, null, null);
+ null, null, null, null, null, null, null, null, null);
assertEquals(1, entities.size());
for (TimelineEntity e : entities) {
FlowActivityEntity flowActivity = (FlowActivityEntity)e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd369a54/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
index 5da0192..57c326b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/flow/TestHBaseStorageFlowRun.java
@@ -375,8 +375,7 @@ public class TestHBaseStorageFlowRun {
Set<TimelineEntity> entities = hbr.getEntities(user, cluster, flow, runid,
null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null,
- null, null, null, null, null, null, null, null, null,
- metricsToRetrieve, null);
+ null, null, null, null, null, null, null, metricsToRetrieve, null);
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
Set<TimelineMetric> timelineMetrics = timelineEntity.getMetrics();
@@ -444,7 +443,7 @@ public class TestHBaseStorageFlowRun {
hbr.start();
Set<TimelineEntity> entities = hbr.getEntities(user, cluster, flow, runid,
null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null,
- null, null, null, null, null, null, null, null, null, null, null);
+ null, null, null, null, null, null, null, null, null);
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
assertEquals(0, timelineEntity.getMetrics().size());
@@ -452,8 +451,8 @@ public class TestHBaseStorageFlowRun {
entities = hbr.getEntities(user, cluster, flow, runid,
null, TimelineEntityType.YARN_FLOW_RUN.toString(), null, null, null,
- null, null, null, null, null, null, null, null, null,
- null, EnumSet.of(Field.METRICS));
+ null, null, null, null, null, null, null, null,
+ EnumSet.of(Field.METRICS));
assertEquals(1, entities.size());
for (TimelineEntity timelineEntity : entities) {
Set<TimelineMetric> timelineMetrics = timelineEntity.getMetrics();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: YARN-4224. Support fetching
entities by UID and change the REST interface to conform to current REST
APIs' in YARN. (Varun Saxena via gtcarrera9)
Posted by gt...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
new file mode 100644
index 0000000..d052d51
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineReaderWebServicesUtils.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.io.IOException;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import javax.servlet.http.HttpServletRequest;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
+import org.apache.hadoop.yarn.server.timelineservice.storage.TimelineReader.Field;
+
+/**
+ * Set of utility methods to be used by timeline reader web services.
+ */
+final class TimelineReaderWebServicesUtils {
+ private TimelineReaderWebServicesUtils() {
+ }
+
+ /**
+ * Parse a delimited string and convert it into a set of strings. For
+ * instance, if delimiter is ",", then the string should be represented as
+ * "value1,value2,value3".
+ * @param str delimited string.
+ * @param delimiter string is delimited by this delimiter.
+ * @return set of strings.
+ */
+ static Set<String> parseValuesStr(String str, String delimiter) {
+ if (str == null || str.isEmpty()) {
+ return null;
+ }
+ Set<String> strSet = new HashSet<String>();
+ String[] strs = str.split(delimiter);
+ for (String aStr : strs) {
+ strSet.add(aStr.trim());
+ }
+ return strSet;
+ }
+
+ @SuppressWarnings("unchecked")
+ private static <T> void parseKeyValues(Map<String, T> map, String str,
+ String pairsDelim, String keyValuesDelim, boolean stringValue,
+ boolean multipleValues) {
+ String[] pairs = str.split(pairsDelim);
+ for (String pair : pairs) {
+ if (pair == null || pair.trim().isEmpty()) {
+ continue;
+ }
+ String[] pairStrs = pair.split(keyValuesDelim);
+ if (pairStrs.length < 2) {
+ continue;
+ }
+ if (!stringValue) {
+ try {
+ Object value =
+ GenericObjectMapper.OBJECT_READER.readValue(pairStrs[1].trim());
+ map.put(pairStrs[0].trim(), (T) value);
+ } catch (IOException e) {
+ map.put(pairStrs[0].trim(), (T) pairStrs[1].trim());
+ }
+ } else {
+ String key = pairStrs[0].trim();
+ if (multipleValues) {
+ Set<String> values = new HashSet<String>();
+ for (int i = 1; i < pairStrs.length; i++) {
+ values.add(pairStrs[i].trim());
+ }
+ map.put(key, (T) values);
+ } else {
+ map.put(key, (T) pairStrs[1].trim());
+ }
+ }
+ }
+ }
+
+ /**
+ * Parse a delimited string and convert it into a map of key-values with each
+ * key having a set of values. Both the key and values are interpreted as
+ * strings.
+ * For instance, if pairsDelim is "," and keyValuesDelim is ":", then the
+ * string should be represented as
+ * "key1:value11:value12:value13,key2:value21,key3:value31:value32".
+ * @param str delimited string represented as multiple keys having multiple
+ * values.
+ * @param pairsDelim key-values pairs are delimited by this delimiter.
+ * @param keyValuesDelim values for a key are delimited by this delimiter.
+ * @return a map of key-values with each key having a set of values.
+ */
+ static Map<String, Set<String>> parseKeyStrValuesStr(String str,
+ String pairsDelim, String keyValuesDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, Set<String>> map = new HashMap<String, Set<String>>();
+ parseKeyValues(map, str, pairsDelim, keyValuesDelim, true, true);
+ return map;
+ }
+
+ /**
+ * Parse a delimited string and convert it into a map of key-value pairs with
+ * both the key and value interpreted as strings.
+ * For instance, if pairsDelim is "," and keyValDelim is ":", then the string
+ * should be represented as "key1:value1,key2:value2,key3:value3".
+ * @param str delimited string represented as key-value pairs.
+ * @param pairsDelim key-value pairs are delimited by this delimiter.
+ * @param keyValuesDelim key and value are delimited by this delimiter.
+ * @return a map of key-value pairs with both key and value being strings.
+ */
+ static Map<String, String> parseKeyStrValueStr(String str,
+ String pairsDelim, String keyValDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, String> map = new HashMap<String, String>();
+ parseKeyValues(map, str, pairsDelim, keyValDelim, true, false);
+ return map;
+ }
+
+ /**
+ * Parse a delimited string and convert it into a map of key-value pairs with
+ * key being a string and value interpreted as any object.
+ * For instance, if pairsDelim is "," and keyValDelim is ":", then the string
+ * should be represented as "key1:value1,key2:value2,key3:value3".
+ * @param str delimited string represented as key-value pairs.
+ * @param pairsDelim key-value pairs are delimited by this delimiter.
+ * @param keyValuesDelim key and value are delimited by this delimiter.
+ * @return a map of key-value pairs with key being a string and value amy
+ * object.
+ */
+ static Map<String, Object> parseKeyStrValueObj(String str,
+ String pairsDelim, String keyValDelim) {
+ if (str == null) {
+ return null;
+ }
+ Map<String, Object> map = new HashMap<String, Object>();
+ parseKeyValues(map, str, pairsDelim, keyValDelim, false, false);
+ return map;
+ }
+
+ /**
+ * Interprets passed string as set of fields delimited by passed delimiter.
+ * For instance, if delimiter is ",", then the passed string should be
+ * represented as "METRICS,CONFIGS" where the delimited parts of the string
+ * present in {@link Field}.
+ * @param str passed string.
+ * @param delimiter string delimiter.
+ * @return a set of {@link Field}.
+ */
+ static EnumSet<Field> parseFieldsStr(String str, String delimiter) {
+ if (str == null) {
+ return null;
+ }
+ String[] strs = str.split(delimiter);
+ EnumSet<Field> fieldList = EnumSet.noneOf(Field.class);
+ for (String s : strs) {
+ fieldList.add(Field.valueOf(s.trim().toUpperCase()));
+ }
+ return fieldList;
+ }
+
+ /**
+ * Interpret passed string as a long.
+ * @param str Passed string.
+ * @return long representation if string is not null, null otherwise.
+ */
+ static Long parseLongStr(String str) {
+ return str == null ? null : Long.parseLong(str.trim());
+ }
+
+ /**
+ * Trims the passed string if its not null.
+ * @param str Passed string.
+ * @return trimmed string if string is not null, null otherwise.
+ */
+ static String parseStr(String str) {
+ return str == null ? null : str.trim();
+ }
+
+ /**
+ * Get UGI from HTTP request.
+ * @param req HTTP request.
+ * @return UGI.
+ */
+ static UserGroupInformation getUser(HttpServletRequest req) {
+ String remoteUser = req.getRemoteUser();
+ UserGroupInformation callerUGI = null;
+ if (remoteUser != null) {
+ callerUGI = UserGroupInformation.createRemoteUser(remoteUser);
+ }
+ return callerUGI;
+ }
+
+ /**
+ * Get username from caller UGI.
+ * @param callerUGI caller UGI.
+ * @return username.
+ */
+ static String getUserName(UserGroupInformation callerUGI) {
+ return ((callerUGI != null) ? callerUGI.getUserName().trim() : "");
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
new file mode 100644
index 0000000..cac3f06
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/reader/TimelineUIDConverter.java
@@ -0,0 +1,245 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import java.util.List;
+
+/**
+ * Used for encoding/decoding UID which will be used for query by UI.
+ */
+enum TimelineUIDConverter {
+ // Flow UID should contain cluster, user and flow name.
+ FLOW_UID {
+ @Override
+ String encodeUID(TimelineReaderContext context) {
+ if (context == null) {
+ return null;
+ }
+ if (context.getClusterId() == null || context.getUserId() == null ||
+ context.getFlowName() == null) {
+ return null;
+ }
+ String[] flowNameTupleArr = {context.getClusterId(), context.getUserId(),
+ context.getFlowName()};
+ return joinAndEscapeUIDParts(flowNameTupleArr);
+ }
+
+ @Override
+ TimelineReaderContext decodeUID(String uId) throws Exception {
+ if (uId == null) {
+ return null;
+ }
+ List<String> flowNameTupleList = splitUID(uId);
+ // Should have 3 parts i.e. cluster, user and flow name.
+ if (flowNameTupleList.size() != 3) {
+ return null;
+ }
+ return new TimelineReaderContext(flowNameTupleList.get(0),
+ flowNameTupleList.get(1), flowNameTupleList.get(2), null,
+ null, null, null);
+ }
+ },
+
+ // Flowrun UID should contain cluster, user, flow name and flowrun id.
+ FLOWRUN_UID{
+ @Override
+ String encodeUID(TimelineReaderContext context) {
+ if (context == null) {
+ return null;
+ }
+ if (context.getClusterId() == null || context.getUserId() == null ||
+ context.getFlowName() == null || context.getFlowRunId() == null) {
+ return null;
+ }
+ String[] flowRunTupleArr = {context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId().toString()};
+ return joinAndEscapeUIDParts(flowRunTupleArr);
+ }
+
+ @Override
+ TimelineReaderContext decodeUID(String uId) throws Exception {
+ if (uId == null) {
+ return null;
+ }
+ List<String> flowRunTupleList = splitUID(uId);
+ // Should have 4 parts i.e. cluster, user, flow name and flowrun id.
+ if (flowRunTupleList.size() != 4) {
+ return null;
+ }
+ return new TimelineReaderContext(flowRunTupleList.get(0),
+ flowRunTupleList.get(1), flowRunTupleList.get(2),
+ Long.parseLong(flowRunTupleList.get(3)), null, null, null);
+ }
+ },
+
+ // Application UID should contain cluster, user, flow name, flowrun id
+ // and app id OR cluster and app id(i.e.without flow context info).
+ APPLICATION_UID{
+ @Override
+ String encodeUID(TimelineReaderContext context) {
+ if (context == null) {
+ return null;
+ }
+ if (context.getClusterId() == null || context.getAppId() == null) {
+ return null;
+ }
+ if (context.getUserId() != null && context.getFlowName() != null &&
+ context.getFlowRunId() != null) {
+ // Flow information exists.
+ String[] appTupleArr = {context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId().toString(),
+ context.getAppId()};
+ return joinAndEscapeUIDParts(appTupleArr);
+ } else {
+ // Only cluster and app information exists. Flow info does not exist.
+ String[] appTupleArr = {context.getClusterId(), context.getAppId()};
+ return joinAndEscapeUIDParts(appTupleArr);
+ }
+ }
+
+ @Override
+ TimelineReaderContext decodeUID(String uId) throws Exception {
+ if (uId == null) {
+ return null;
+ }
+ List<String> appTupleList = splitUID(uId);
+ // Should have 5 parts i.e. cluster, user, flow name, flowrun id
+ // and app id OR should have 2 parts i.e. cluster and app id.
+ if (appTupleList.size() == 5) {
+ // Flow information exists.
+ return new TimelineReaderContext(appTupleList.get(0),
+ appTupleList.get(1), appTupleList.get(2),
+ Long.parseLong(appTupleList.get(3)), appTupleList.get(4),
+ null, null);
+ } else if (appTupleList.size() == 2) {
+ // Flow information does not exist.
+ return new TimelineReaderContext(appTupleList.get(0), null, null, null,
+ appTupleList.get(1), null, null);
+ } else {
+ return null;
+ }
+ }
+ },
+
+ // Generic Entity UID should contain cluster, user, flow name, flowrun id,
+ // app id, entity type and entity id OR should contain cluster, appid, entity
+ // type and entity id(i.e.without flow context info).
+ GENERIC_ENTITY_UID {
+ @Override
+ String encodeUID(TimelineReaderContext context) {
+ if (context == null) {
+ return null;
+ }
+ if (context.getClusterId() == null || context.getAppId() == null ||
+ context.getEntityType() == null || context.getEntityId() == null) {
+ return null;
+ }
+ if (context.getUserId() != null && context.getFlowName() != null &&
+ context.getFlowRunId() != null) {
+ // Flow information exists.
+ String[] entityTupleArr = {context.getClusterId(), context.getUserId(),
+ context.getFlowName(), context.getFlowRunId().toString(),
+ context.getAppId(), context.getEntityType(), context.getEntityId()};
+ return joinAndEscapeUIDParts(entityTupleArr);
+ } else {
+ // Only entity and app information exists. Flow info does not exist.
+ String[] entityTupleArr = {context.getClusterId(), context.getAppId(),
+ context.getEntityType(), context.getEntityId()};
+ return joinAndEscapeUIDParts(entityTupleArr);
+ }
+ }
+
+ @Override
+ TimelineReaderContext decodeUID(String uId) throws Exception {
+ if (uId == null) {
+ return null;
+ }
+ List<String> entityTupleList = splitUID(uId);
+ // Should have 7 parts i.e. cluster, user, flow name, flowrun id, app id,
+ // entity type and entity id OR should have 4 parts i.e. cluster, app id,
+ // entity type and entity id.
+ if (entityTupleList.size() == 7) {
+ // Flow information exists.
+ return new TimelineReaderContext(entityTupleList.get(0),
+ entityTupleList.get(1), entityTupleList.get(2),
+ Long.parseLong(entityTupleList.get(3)), entityTupleList.get(4),
+ entityTupleList.get(5), entityTupleList.get(6));
+ } else if (entityTupleList.size() == 4) {
+ // Flow information does not exist.
+ return new TimelineReaderContext(entityTupleList.get(0), null, null,
+ null, entityTupleList.get(1), entityTupleList.get(2),
+ entityTupleList.get(3));
+ } else {
+ return null;
+ }
+ }
+ };
+
+ /**
+ * Delimiter used for UID.
+ */
+ public static final char UID_DELIMITER_CHAR = '!';
+
+ /**
+ * Escape Character used if delimiter or escape character itself is part of
+ * different components of UID.
+ */
+ public static final char UID_ESCAPE_CHAR = '*';
+
+ /**
+ * Split UID using {@link #UID_DELIMITER_CHAR} and {@link #UID_ESCAPE_CHAR}.
+ * @param uid
+ * @return a list of different parts of UID split across delimiter.
+ * @throws IllegalArgumentException if UID is not properly escaped.
+ */
+ private static List<String> splitUID(String uid)
+ throws IllegalArgumentException {
+ return TimelineReaderUtils.split(uid, UID_DELIMITER_CHAR, UID_ESCAPE_CHAR);
+ }
+
+ /**
+ * Join different parts of UID delimited by {@link #UID_DELIMITER_CHAR} with
+ * delimiter and escape character escaped using {@link #UID_ESCAPE_CHAR} if
+ * UID parts contain them.
+ * @param parts an array of UID parts to be joined.
+ * @return a string joined using the delimiter with escape and delimiter
+ * characters escaped if they are part of the string parts to be joined.
+ * Returns null if one of the parts is null.
+ */
+ private static String joinAndEscapeUIDParts(String[] parts) {
+ return TimelineReaderUtils.joinAndEscapeStrings(parts, UID_DELIMITER_CHAR,
+ UID_ESCAPE_CHAR);
+ }
+
+ /**
+ * Encodes UID depending on UID implementation.
+ * @param context
+ * @return UID represented as a string.
+ */
+ abstract String encodeUID(TimelineReaderContext context);
+
+ /**
+ * Decodes UID depending on UID implementation.
+ * @param uId
+ * @return a {@link TimelineReaderContext} object if UID passed can be
+ * decoded, null otherwise.
+ * @throws Exception
+ */
+ abstract TimelineReaderContext decodeUID(String uId) throws Exception;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
index a5a271e..0eeb195 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/TimelineReader.java
@@ -26,6 +26,8 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowActivityEntity;
+import org.apache.hadoop.yarn.api.records.timelineservice.FlowRunEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelineFilterList;
import org.apache.hadoop.yarn.server.timelineservice.reader.filter.TimelinePrefixFilter;
@@ -87,10 +89,14 @@ public interface TimelineReader extends Service {
* @param fieldsToRetrieve
* Specifies which fields of the entity object to retrieve(optional), see
* {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type and entity created time. All entities will be returned if
+ * entity type and entity created time. All fields will be returned if
* {@link Field#ALL} is specified.
* @return a {@link TimelineEntity} instance or null. The entity will
* contain the metadata plus the given fields to retrieve.
+ * If entityType is YARN_FLOW_RUN, entity returned is of type
+ * {@link FlowRunEntity}.
+ * For all other entity types, entity returned is of type
+ * {@link TimelineEntity}.
* @throws IOException
*/
TimelineEntity getEntity(String userId, String clusterId, String flowName,
@@ -167,12 +173,18 @@ public interface TimelineReader extends Service {
* @param fieldsToRetrieve
* Specifies which fields of the entity object to retrieve(optional), see
* {@link Field}. If null, retrieves 4 fields namely entity id,
- * entity type and entity created time. All entities will be returned if
+ * entity type and entity created time. All fields will be returned if
* {@link Field#ALL} is specified.
* @return A set of {@link TimelineEntity} instances of the given entity type
* in the given context scope which matches the given predicates
* ordered by created time, descending. Each entity will only contain the
* metadata(id, type and created time) plus the given fields to retrieve.
+ * If entityType is YARN_FLOW_ACTIVITY, entities returned are of type
+ * {@link FlowActivityEntity}.
+ * If entityType is YARN_FLOW_RUN, entities returned are of type
+ * {@link FlowRunEntity}.
+ * For all other entity types, entities returned are of type
+ * {@link TimelineEntity}.
* @throws IOException
*/
Set<TimelineEntity> getEntities(String userId, String clusterId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
index 990856f..237b9ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/main/java/org/apache/hadoop/yarn/server/timelineservice/storage/reader/GenericEntityReader.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumn
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityColumnPrefix;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityRowKey;
import org.apache.hadoop.yarn.server.timelineservice.storage.entity.EntityTable;
+import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.common.base.Preconditions;
@@ -195,9 +196,9 @@ class GenericEntityReader extends TimelineEntityReader {
AppToFlowColumn.FLOW_ID.readResult(result).toString(),
((Number)AppToFlowColumn.FLOW_RUN_ID.readResult(result)).longValue());
} else {
- throw new IOException(
- "Unable to find the context flow ID and flow run ID for clusterId=" +
- clusterId + ", appId=" + appId);
+ throw new NotFoundException(
+ "Unable to find the context flow ID and flow run ID for clusterId=" +
+ clusterId + ", appId=" + appId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java
new file mode 100644
index 0000000..791d6ab
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderUtils.java
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.util.List;
+
+import org.junit.Test;
+
+public class TestTimelineReaderUtils {
+
+ @Test
+ public void testSplitUsingEscapeAndDelimChar() throws Exception {
+ List<String> list =
+ TimelineReaderUtils.split("*!cluster!*!b**o***!xer!oozie**", '!', '*');
+ String[] arr = new String[list.size()];
+ arr = list.toArray(arr);
+ assertArrayEquals(new String[] {"!cluster", "!b*o*!xer", "oozie*"}, arr);
+ list = TimelineReaderUtils.split("*!cluster!*!b**o***!xer!!", '!', '*');
+ arr = new String[list.size()];
+ arr = list.toArray(arr);
+ assertArrayEquals(new String[] {"!cluster", "!b*o*!xer", "", ""}, arr);
+ }
+
+ @Test
+ public void testJoinAndEscapeStrings() throws Exception {
+ assertEquals("*!cluster!*!b**o***!xer!oozie**",
+ TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] { "!cluster", "!b*o*!xer", "oozie*"}, '!', '*'));
+ assertEquals("*!cluster!*!b**o***!xer!!",
+ TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] { "!cluster", "!b*o*!xer", "", ""}, '!', '*'));
+ assertNull(TimelineReaderUtils.joinAndEscapeStrings(
+ new String[] { "!cluster", "!b*o*!xer", null, ""}, '!', '*'));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
index 1aefdca..b6e23a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServices.java
@@ -166,7 +166,7 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/app1/app/id_1");
+ "timeline/clusters/cluster1/apps/app1/entities/app/id_1");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -188,8 +188,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/app1/app/id_1?userid=user1&" +
- "flowname=flow1&flowrunid=1");
+ "timeline/clusters/cluster1/apps/app1/entities/app/id_1?" +
+ "userid=user1&flowname=flow1&flowrunid=1");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -208,7 +208,8 @@ public class TestTimelineReaderWebServices {
try {
// Fields are case insensitive.
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/app1/app/id_1?fields=CONFIGS,Metrics,info");
+ "timeline/clusters/cluster1/apps/app1/entities/app/id_1?" +
+ "fields=CONFIGS,Metrics,info");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -217,7 +218,10 @@ public class TestTimelineReaderWebServices {
assertEquals("app", entity.getType());
assertEquals(3, entity.getConfigs().size());
assertEquals(3, entity.getMetrics().size());
- assertEquals(1, entity.getInfo().size());
+ assertTrue("UID should be present",
+ entity.getInfo().containsKey(TimelineReaderManager.UID_KEY));
+ // Includes UID.
+ assertEquals(2, entity.getInfo().size());
// No events will be returned as events are not part of fields.
assertEquals(0, entity.getEvents().size());
} finally {
@@ -230,7 +234,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/app1/app/id_1?fields=ALL");
+ "timeline/clusters/cluster1/apps/app1/entities/app/id_1?" +
+ "fields=ALL");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -239,7 +244,10 @@ public class TestTimelineReaderWebServices {
assertEquals("app", entity.getType());
assertEquals(3, entity.getConfigs().size());
assertEquals(3, entity.getMetrics().size());
- assertEquals(1, entity.getInfo().size());
+ assertTrue("UID should be present",
+ entity.getInfo().containsKey(TimelineReaderManager.UID_KEY));
+ // Includes UID.
+ assertEquals(2, entity.getInfo().size());
assertEquals(2, entity.getEvents().size());
} finally {
client.destroy();
@@ -251,7 +259,7 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/app1/app/id_10");
+ "timeline/clusters/cluster1/apps/app1/entities/app/id_10");
verifyHttpResponse(client, uri, Status.NOT_FOUND);
} finally {
client.destroy();
@@ -263,7 +271,7 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/app1/app/id_1");
+ "timeline/apps/app1/entities/app/id_1");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -272,7 +280,7 @@ public class TestTimelineReaderWebServices {
assertEquals("app", entity.getType());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/app1/app");
+ "timeline/apps/app1/entities/app");
resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -289,7 +297,7 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app");
+ "timeline/clusters/cluster1/apps/app1/entities/app");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -312,7 +320,7 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?limit=2");
+ "timeline/clusters/cluster1/apps/app1/entities/app?limit=2");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -326,7 +334,7 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_4")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?limit=3");
+ "clusters/cluster1/apps/app1/entities/app?limit=3");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -344,8 +352,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?createdtimestart=1425016502030&"
- + "createdtimeend=1425016502060");
+ "timeline/clusters/cluster1/apps/app1/entities/app?" +
+ "createdtimestart=1425016502030&createdtimeend=1425016502060");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -356,7 +364,8 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_4")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?createdtimeend=1425016502010");
+ "clusters/cluster1/apps/app1/entities/app?createdtimeend" +
+ "=1425016502010");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -366,7 +375,8 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_4")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?createdtimestart=1425016502010");
+ "clusters/cluster1/apps/app1/entities/app?createdtimestart=" +
+ "1425016502010");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -384,7 +394,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?relatesto=flow:flow1");
+ "timeline/clusters/cluster1/apps/app1/entities/app?relatesto=" +
+ "flow:flow1");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -395,8 +406,8 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_1")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?isrelatedto=type1:tid1_2,type2:" +
- "tid2_1%60");
+ "clusters/cluster1/apps/app1/entities/app?isrelatedto=" +
+ "type1:tid1_2,type2:tid2_1%60");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -406,8 +417,8 @@ public class TestTimelineReaderWebServices {
entities.contains(newEntity("app", "id_1")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app?isrelatedto=type1:tid1_1:tid1_2" +
- ",type2:tid2_1%60");
+ "clusters/cluster1/apps/app1/entities/app?isrelatedto=" +
+ "type1:tid1_1:tid1_2,type2:tid2_1%60");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -425,8 +436,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?conffilters=config_1:123," +
- "config_3:abc");
+ "timeline/clusters/cluster1/apps/app1/entities/app?" +
+ "conffilters=config_1:123,config_3:abc");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -447,7 +458,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?infofilters=info2:3.5");
+ "timeline/clusters/cluster1/apps/app1/entities/app?" +
+ "infofilters=info2:3.5");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -466,7 +478,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?metricfilters=metric3");
+ "timeline/clusters/cluster1/apps/app1/entities/app?" +
+ "metricfilters=metric3");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -487,7 +500,8 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?eventfilters=event_2,event_4");
+ "timeline/clusters/cluster1/apps/app1/entities/app?" +
+ "eventfilters=event_2,event_4");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -506,10 +520,11 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?metricfilters=metric7&" +
- "isrelatedto=type1:tid1_1;tid1_2,type2:tid2_1%60&relatesto=" +
- "flow:flow1&eventfilters=event_2,event_4&infofilters=info2:3.5" +
- "&createdtimestart=1425016502030&createdtimeend=1425016502060");
+ "timeline/clusters/cluster1/apps/app1/entities/app?" +
+ "metricfilters=metric7&isrelatedto=type1:tid1_1;tid1_2,type2:tid2_1" +
+ "%60&relatesto=flow:flow1&eventfilters=event_2,event_4&infofilters=" +
+ "info2:3.5&createdtimestart=1425016502030&createdtimeend=" +
+ "1425016502060");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -526,15 +541,15 @@ public class TestTimelineReaderWebServices {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/app1/app?flowrunid=a23b");
+ "timeline/clusters/cluster1/apps/app1/entities/app?flowrunid=a23b");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entity/cluster1/app1/app/id_1?flowrunid=2ab15");
+ "clusters/cluster1/apps/app1/entities/app/id_1?flowrunid=2ab15");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
- "entities/cluster1/app1/app/?limit=#$561av");
+ "clusters/cluster1/apps/app1/entities/app?limit=#$561av");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
} finally {
client.destroy();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
index 0918d40..9eaa3de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineReaderWebServicesHBaseStorage.java
@@ -28,8 +28,10 @@ import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
import java.text.DateFormat;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -349,7 +351,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrun/user1/cluster1/flow_name/1002345678919");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678919");
ClientResponse resp = getResponse(client, uri);
FlowRunEntity entity = resp.getEntity(FlowRunEntity.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -366,7 +369,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
// Query without specifying cluster ID.
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrun/user1/flow_name/1002345678919");
+ "timeline/users/user1/flows/flow_name/runs/1002345678919");
resp = getResponse(client, uri);
entity = resp.getEntity(FlowRunEntity.class);
assertNotNull(entity);
@@ -390,7 +393,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowruns/user1/cluster1/flow_name");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs");
ClientResponse resp = getResponse(client, uri);
Set<FlowRunEntity> entities =
resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
@@ -408,8 +411,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertEquals(0, entity.getMetrics().size());
}
- uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowruns/user1/cluster1/flow_name?limit=1");
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "clusters/cluster1/users/user1/flows/flow_name/runs?limit=1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -424,7 +427,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowruns/user1/cluster1/flow_name?" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" +
"createdtimestart=1425016501030");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
@@ -440,7 +443,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowruns/user1/cluster1/flow_name?" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" +
"createdtimestart=1425016500999&createdtimeend=1425016501035");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
@@ -459,7 +462,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowruns/user1/cluster1/flow_name?" +
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" +
"createdtimeend=1425016501030");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
@@ -475,7 +478,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowruns/user1/cluster1/flow_name?fields=metrics");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs?" +
+ "fields=metrics");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
assertEquals(MediaType.APPLICATION_JSON_TYPE, resp.getType());
@@ -498,11 +502,262 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
@Test
+ public void testGetEntitiesByUID() throws Exception {
+ Client client = createClient();
+ try {
+ // Query all flows.
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flows");
+ ClientResponse resp = getResponse(client, uri);
+ Set<FlowActivityEntity> flowEntities =
+ resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
+ assertNotNull(flowEntities);
+ assertEquals(2, flowEntities.size());
+ List<String> listFlowUIDs = new ArrayList<String>();
+ for (FlowActivityEntity entity : flowEntities) {
+ String flowUID =
+ (String)entity.getInfo().get(TimelineReaderManager.UID_KEY);
+ listFlowUIDs.add(flowUID);
+ assertEquals(TimelineUIDConverter.FLOW_UID.encodeUID(
+ new TimelineReaderContext(entity.getCluster(), entity.getUser(),
+ entity.getFlowName(), null, null, null, null)), flowUID);
+ assertTrue((entity.getId().endsWith("@flow_name") &&
+ entity.getFlowRuns().size() == 2) ||
+ (entity.getId().endsWith("@flow_name2") &&
+ entity.getFlowRuns().size() == 1));
+ }
+
+ // Query flowruns based on UID returned in query above.
+ List<String> listFlowRunUIDs = new ArrayList<String>();
+ for (String flowUID : listFlowUIDs) {
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flow-uid/" + flowUID + "/runs");
+ resp = getResponse(client, uri);
+ Set<FlowRunEntity> frEntities =
+ resp.getEntity(new GenericType<Set<FlowRunEntity>>(){});
+ assertNotNull(frEntities);
+ for (FlowRunEntity entity : frEntities) {
+ String flowRunUID =
+ (String)entity.getInfo().get(TimelineReaderManager.UID_KEY);
+ listFlowRunUIDs.add(flowRunUID);
+ assertEquals(TimelineUIDConverter.FLOWRUN_UID.encodeUID(
+ new TimelineReaderContext("cluster1", entity.getUser(),
+ entity.getName(), entity.getRunId(), null, null, null)),
+ flowRunUID);
+ }
+ }
+ assertEquals(3, listFlowRunUIDs.size());
+
+ // Query single flowrun based on UIDs' returned in query to get flowruns.
+ for (String flowRunUID : listFlowRunUIDs) {
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/run-uid/" + flowRunUID);
+ resp = getResponse(client, uri);
+ FlowRunEntity entity = resp.getEntity(FlowRunEntity.class);
+ assertNotNull(entity);
+ }
+
+ // Query apps based on UIDs' returned in query to get flowruns.
+ List<String> listAppUIDs = new ArrayList<String>();
+ for (String flowRunUID : listFlowRunUIDs) {
+ TimelineReaderContext context =
+ TimelineUIDConverter.FLOWRUN_UID.decodeUID(flowRunUID);
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/run-uid/" + flowRunUID + "/apps");
+ resp = getResponse(client, uri);
+ Set<TimelineEntity> appEntities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(appEntities);
+ for (TimelineEntity entity : appEntities) {
+ String appUID =
+ (String)entity.getInfo().get(TimelineReaderManager.UID_KEY);
+ listAppUIDs.add(appUID);
+ assertEquals(TimelineUIDConverter.APPLICATION_UID.encodeUID(
+ new TimelineReaderContext(context.getClusterId(),
+ context.getUserId(), context.getFlowName(),
+ context.getFlowRunId(), entity.getId(), null, null)), appUID);
+ }
+ }
+ assertEquals(4, listAppUIDs.size());
+
+ // Query single app based on UIDs' returned in query to get apps.
+ for (String appUID : listAppUIDs) {
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app-uid/" + appUID);
+ resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ }
+
+ // Query entities based on UIDs' returned in query to get apps and
+ // a specific entity type(in this case type1).
+ List<String> listEntityUIDs = new ArrayList<String>();
+ for (String appUID : listAppUIDs) {
+ TimelineReaderContext context =
+ TimelineUIDConverter.APPLICATION_UID.decodeUID(appUID);
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app-uid/" + appUID + "/entities/type1");
+ resp = getResponse(client, uri);
+ Set<TimelineEntity> entities =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities);
+ for (TimelineEntity entity : entities) {
+ String entityUID =
+ (String)entity.getInfo().get(TimelineReaderManager.UID_KEY);
+ listEntityUIDs.add(entityUID);
+ assertEquals(TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(
+ new TimelineReaderContext(context.getClusterId(),
+ context.getUserId(), context.getFlowName(),
+ context.getFlowRunId(), context.getAppId(), "type1",
+ entity.getId())), entityUID);
+ }
+ }
+ assertEquals(2, listEntityUIDs.size());
+
+ // Query single entity based on UIDs' returned in query to get entities.
+ for (String entityUID : listEntityUIDs) {
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity-uid/" + entityUID);
+ resp = getResponse(client, uri);
+ TimelineEntity entity = resp.getEntity(TimelineEntity.class);
+ assertNotNull(entity);
+ }
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/flow-uid/dummy:flow/runs");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/run-uid/dummy:flowrun");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ // Run Id is not a numerical value.
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/run-uid/some:dummy:flow:123v456");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/run-uid/dummy:flowrun/apps");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app-uid/dummy:app");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/app-uid/dummy:app/entities/type1");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
+ "timeline/entity-uid/dummy:entity");
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testUIDQueryWithAndWithoutFlowContextInfo() throws Exception {
+ Client client = createClient();
+ try {
+ String appUIDWithFlowInfo =
+ "cluster1!user1!flow_name!1002345678919!application_1111111111_1111";
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"+
+ "timeline/app-uid/" + appUIDWithFlowInfo);
+ ClientResponse resp = getResponse(client, uri);
+ TimelineEntity appEntity1 = resp.getEntity(TimelineEntity.class);
+ assertNotNull(appEntity1);
+ assertEquals(
+ TimelineEntityType.YARN_APPLICATION.toString(), appEntity1.getType());
+ assertEquals("application_1111111111_1111", appEntity1.getId());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "app-uid/" + appUIDWithFlowInfo + "/entities/type1");
+ resp = getResponse(client, uri);
+ Set<TimelineEntity> entities1 =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities1);
+ assertEquals(2, entities1.size());
+ for (TimelineEntity entity : entities1) {
+ assertNotNull(entity.getInfo());
+ assertEquals(1, entity.getInfo().size());
+ String uid =
+ (String) entity.getInfo().get(TimelineReaderManager.UID_KEY);
+ assertNotNull(uid);
+ assertTrue(uid.equals(appUIDWithFlowInfo + "!type1!entity1") ||
+ uid.equals(appUIDWithFlowInfo + "!type1!entity2"));
+ }
+
+ String appUIDWithoutFlowInfo = "cluster1!application_1111111111_1111";
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
+ "app-uid/" + appUIDWithoutFlowInfo);
+ resp = getResponse(client, uri);;
+ TimelineEntity appEntity2 = resp.getEntity(TimelineEntity.class);
+ assertNotNull(appEntity2);
+ assertEquals(
+ TimelineEntityType.YARN_APPLICATION.toString(), appEntity2.getType());
+ assertEquals("application_1111111111_1111", appEntity2.getId());
+
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/" +
+ "app-uid/" + appUIDWithoutFlowInfo + "/entities/type1");
+ resp = getResponse(client, uri);
+ Set<TimelineEntity> entities2 =
+ resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
+ assertNotNull(entities2);
+ assertEquals(2, entities2.size());
+ for (TimelineEntity entity : entities2) {
+ assertNotNull(entity.getInfo());
+ assertEquals(1, entity.getInfo().size());
+ String uid =
+ (String) entity.getInfo().get(TimelineReaderManager.UID_KEY);
+ assertNotNull(uid);
+ assertTrue(uid.equals(appUIDWithoutFlowInfo + "!type1!entity1") ||
+ uid.equals(appUIDWithoutFlowInfo + "!type1!entity2"));
+ }
+
+ String entityUIDWithFlowInfo = appUIDWithFlowInfo + "!type1!entity1";
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
+ "entity-uid/" + entityUIDWithFlowInfo);
+ resp = getResponse(client, uri);;
+ TimelineEntity singleEntity1 = resp.getEntity(TimelineEntity.class);
+ assertNotNull(singleEntity1);
+ assertEquals("type1", singleEntity1.getType());
+ assertEquals("entity1", singleEntity1.getId());
+
+ String entityUIDWithoutFlowInfo =
+ appUIDWithoutFlowInfo + "!type1!entity1";
+ uri = URI.create("http://localhost:" + serverPort + "/ws/v2/timeline/"+
+ "entity-uid/" + entityUIDWithoutFlowInfo);
+ resp = getResponse(client, uri);;
+ TimelineEntity singleEntity2 = resp.getEntity(TimelineEntity.class);
+ assertNotNull(singleEntity2);
+ assertEquals("type1", singleEntity2.getType());
+ assertEquals("entity1", singleEntity2.getId());
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
+ public void testUIDNotProperlyEscaped() throws Exception {
+ Client client = createClient();
+ try {
+ String appUID =
+ "cluster1!user*1!flow_name!1002345678919!application_1111111111_1111";
+ URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/"+
+ "timeline/app-uid/" + appUID);
+ verifyHttpResponse(client, uri, Status.BAD_REQUEST);
+ } finally {
+ client.destroy();
+ }
+ }
+
+ @Test
public void testGetFlows() throws Exception {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1");
+ "timeline/clusters/cluster1/flows");
ClientResponse resp = getResponse(client, uri);
Set<FlowActivityEntity> entities =
resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
@@ -524,7 +779,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertEquals(2, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?limit=1");
+ "timeline/clusters/cluster1/flows?limit=1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
assertNotNull(entities);
@@ -532,8 +787,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
DateFormat fmt = TimelineReaderWebServices.DATE_FORMAT.get();
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=" + fmt.format(dayTs) + "-" +
- fmt.format(dayTs + (2*86400000L)));
+ "timeline/clusters/cluster1/flows?daterange=" + fmt.format(dayTs) +
+ "-" + fmt.format(dayTs + (2*86400000L)));
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
assertNotNull(entities);
@@ -546,7 +801,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=" +
+ "timeline/clusters/cluster1/flows?daterange=" +
fmt.format(dayTs + (4*86400000L)));
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
@@ -554,7 +809,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertEquals(0, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=-" +
+ "timeline/clusters/cluster1/flows?daterange=-" +
fmt.format(dayTs + (2*86400000L)));
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
@@ -562,7 +817,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertEquals(2, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=" +
+ "timeline/clusters/cluster1/flows?daterange=" +
fmt.format(dayTs - (2*86400000L)) + "-");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
@@ -570,19 +825,19 @@ public class TestTimelineReaderWebServicesHBaseStorage {
assertEquals(2, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=20150711:20150714");
+ "timeline/clusters/cluster1/flows?daterange=20150711:20150714");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=20150714-20150711");
+ "timeline/clusters/cluster1/flows?daterange=20150714-20150711");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=2015071129-20150712");
+ "timeline/clusters/cluster1/flows?daterange=2015071129-20150712");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster1?daterange=20150711-2015071243");
+ "timeline/clusters/cluster1/flows?daterange=20150711-2015071243");
verifyHttpResponse(client, uri, Status.BAD_REQUEST);
} finally {
client.destroy();
@@ -594,7 +849,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/app/cluster1/application_1111111111_1111?" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111?" +
"userid=user1&fields=ALL&flowname=flow_name&flowrunid=1002345678919");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
@@ -612,7 +867,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
}
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/app/application_1111111111_2222?userid=user1" +
+ "timeline/apps/application_1111111111_2222?userid=user1" +
"&fields=metrics&flowname=flow_name&flowrunid=1002345678919");
resp = getResponse(client, uri);
entity = resp.getEntity(TimelineEntity.class);
@@ -635,7 +890,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/app/cluster1/application_1111111111_1111?" +
+ "timeline/clusters/cluster1/apps/application_1111111111_1111?" +
"fields=ALL");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
@@ -661,7 +916,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entity/cluster1/application_1111111111_1111/type1/entity1");
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1/entity1");
ClientResponse resp = getResponse(client, uri);
TimelineEntity entity = resp.getEntity(TimelineEntity.class);
assertNotNull(entity);
@@ -677,7 +933,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/entities/cluster1/application_1111111111_1111/type1");
+ "timeline/clusters/cluster1/apps/application_1111111111_1111/" +
+ "entities/type1");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -697,8 +954,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrunapps/user1/cluster1/flow_name/1002345678919?" +
- "fields=ALL");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678919/apps?fields=ALL");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -714,14 +971,15 @@ public class TestTimelineReaderWebServicesHBaseStorage {
// Query without specifying cluster ID.
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrunapps/user1/flow_name/1002345678919");
+ "timeline/users/user1/flows/flow_name/runs/1002345678919/apps");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
assertEquals(2, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrunapps/user1/flow_name/1002345678919?limit=1");
+ "timeline/users/user1/flows/flow_name/runs/1002345678919/" +
+ "apps?limit=1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
@@ -736,7 +994,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/cluster1/flow_name?fields=ALL");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" +
+ "fields=ALL");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -754,14 +1013,14 @@ public class TestTimelineReaderWebServicesHBaseStorage {
// Query without specifying cluster ID.
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/flow_name");
+ "timeline/users/user1/flows/flow_name/apps");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
assertEquals(3, entities.size());
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/flow_name?limit=1");
+ "timeline/users/user1/flows/flow_name/apps?limit=1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
@@ -777,8 +1036,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
try {
String entityType = TimelineEntityType.YARN_APPLICATION.toString();
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/cluster1/flow_name?eventfilters=" +
- ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" +
+ "eventfilters=" + ApplicationMetricsConstants.FINISHED_EVENT_TYPE);
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -788,8 +1047,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
newEntity(entityType, "application_1111111111_1111")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/cluster1/flow_name?metricfilters=" +
- "HDFS_BYTES_READ");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" +
+ "metricfilters=HDFS_BYTES_READ");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
@@ -798,8 +1057,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
newEntity(entityType, "application_1111111111_1111")));
uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/cluster1/flow_name?conffilters=" +
- "cfg1:value1");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/apps?" +
+ "conffilters=cfg1:value1");
resp = getResponse(client, uri);
entities = resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
assertNotNull(entities);
@@ -816,7 +1075,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrun/user1/cluster1/flow_name/1002345678929");
+ "timeline/clusters/cluster1/users/user1/flows/flow_name/runs/" +
+ "1002345678929");
verifyHttpResponse(client, uri, Status.NOT_FOUND);
} finally {
client.destroy();
@@ -828,7 +1088,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flows/cluster2");
+ "timeline/clusters/cluster2/flows");
ClientResponse resp = getResponse(client, uri);
Set<FlowActivityEntity> entities =
resp.getEntity(new GenericType<Set<FlowActivityEntity>>(){});
@@ -845,8 +1105,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/app/user1/cluster1/flow_name/1002345678919/" +
- "application_1111111111_1378");
+ "timeline/clusters/cluster1/apps/application_1111111111_1378");
verifyHttpResponse(client, uri, Status.NOT_FOUND);
} finally {
client.destroy();
@@ -858,7 +1117,8 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowrunapps/user1/cluster2/flow_name/1002345678919");
+ "timeline/clusters/cluster2/users/user1/flows/flow_name/runs/" +
+ "1002345678919/apps");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
@@ -875,7 +1135,7 @@ public class TestTimelineReaderWebServicesHBaseStorage {
Client client = createClient();
try {
URI uri = URI.create("http://localhost:" + serverPort + "/ws/v2/" +
- "timeline/flowapps/user1/cluster2/flow_name55");
+ "timeline/clusters/cluster2/users/user1/flows/flow_name55/apps");
ClientResponse resp = getResponse(client, uri);
Set<TimelineEntity> entities =
resp.getEntity(new GenericType<Set<TimelineEntity>>(){});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java
new file mode 100644
index 0000000..8d6235d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/reader/TestTimelineUIDConverter.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.timelineservice.reader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import org.junit.Test;
+
+public class TestTimelineUIDConverter {
+
+ @Test
+ public void testUIDEncodingDecoding() throws Exception {
+ TimelineReaderContext context = new TimelineReaderContext(
+ "!cluster", "!b*o*!xer", "oozie*", null, null, null, null);
+ String uid = TimelineUIDConverter.FLOW_UID.encodeUID(context);
+ assertEquals("*!cluster!*!b**o***!xer!oozie**", uid);
+ assertEquals(context, TimelineUIDConverter.FLOW_UID.decodeUID(uid));
+
+ context = new TimelineReaderContext("!cluster*", "!b*o!!x!*er", "*oozie!",
+ 123L, null, null, null);
+ uid = TimelineUIDConverter.FLOWRUN_UID.encodeUID(context);
+ assertEquals("*!cluster**!*!b**o*!*!x*!**er!**oozie*!!123", uid);
+ assertEquals(context, TimelineUIDConverter.FLOWRUN_UID.decodeUID(uid));
+
+ context = new TimelineReaderContext("yarn_cluster", "root", "hive_join",
+ 1234L, "application_1111111111_1111", null, null);
+ uid = TimelineUIDConverter.APPLICATION_UID.encodeUID(context);
+ assertEquals(
+ "yarn_cluster!root!hive_join!1234!application_1111111111_1111", uid);
+ assertEquals(context, TimelineUIDConverter.APPLICATION_UID.decodeUID(uid));
+ context = new TimelineReaderContext("yarn_cluster", null, null, null,
+ "application_1111111111_1111", null, null);
+ uid = TimelineUIDConverter.APPLICATION_UID.encodeUID(context);
+ assertEquals("yarn_cluster!application_1111111111_1111", uid);
+ assertEquals(context, TimelineUIDConverter.APPLICATION_UID.decodeUID(uid));
+
+ context = new TimelineReaderContext("yarn_cluster", "root", "hive_join",
+ 1234L, "application_1111111111_1111", "YARN_CONTAINER",
+ "container_1111111111_1111_01_000001");
+ uid = TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context);
+ assertEquals("yarn_cluster!root!hive_join!1234!application_1111111111_1111!"
+ + "YARN_CONTAINER!container_1111111111_1111_01_000001", uid);
+ assertEquals(
+ context, TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uid));
+ context = new TimelineReaderContext("yarn_cluster",null, null, null,
+ "application_1111111111_1111", "YARN_CONTAINER",
+ "container_1111111111_1111_01_000001");
+ uid = TimelineUIDConverter.GENERIC_ENTITY_UID.encodeUID(context);
+ assertEquals("yarn_cluster!application_1111111111_1111!YARN_CONTAINER!" +
+ "container_1111111111_1111_01_000001", uid);
+ assertEquals(
+ context, TimelineUIDConverter.GENERIC_ENTITY_UID.decodeUID(uid));
+ }
+
+ @Test
+ public void testUIDNotProperlyEscaped() throws Exception {
+ try {
+ TimelineUIDConverter.FLOW_UID.decodeUID("*!cluster!*!b*o***!xer!oozie**");
+ fail("UID not properly escaped. Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ TimelineUIDConverter.FLOW_UID.decodeUID("*!cluster!*!b**o***!xer!oozie*");
+ fail("UID not properly escaped. Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {
+ }
+
+ try {
+ TimelineUIDConverter.FLOW_UID.decodeUID("*!cluster!*!b**o***xer!oozie*");
+ fail("UID not properly escaped. Exception should have been thrown.");
+ } catch (IllegalArgumentException e) {
+ }
+ assertNull(TimelineUIDConverter.
+ FLOW_UID.decodeUID("!cluster!*!b**o***!xer!oozie**"));
+ assertNull(TimelineUIDConverter.
+ FLOW_UID.decodeUID("*!cluster!*!b**o**!xer!oozie**"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/97d5cf32/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
index 330adb3..58d5e61 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-timelineservice/src/test/java/org/apache/hadoop/yarn/server/timelineservice/storage/TestPhoenixOfflineAggregationWriterImpl.java
@@ -127,7 +127,7 @@ public class TestPhoenixOfflineAggregationWriterImpl extends BaseTest {
TimelineEntities te = new TimelineEntities();
te.addEntity(getTestAggregationTimelineEntity());
TimelineCollectorContext context = new TimelineCollectorContext("cluster_1",
- "user1", "testFlow", null, 0, null);
+ "user1", "testFlow", null, 0L, null);
storage.writeAggregatedEntity(context, te,
aggregationInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: YARN-4711. NM is going down with
NPE's due to single thread processing of events by Timeline client
(Naganarasimha G R via sjlee)
Posted by gt...@apache.org.
YARN-4711. NM is going down with NPE's due to single thread processing of events by Timeline client (Naganarasimha G R via sjlee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/899468de
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/899468de
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/899468de
Branch: refs/heads/YARN-2928
Commit: 899468de23e065c0d5df3f49470b9358a5ac944b
Parents: b1e7f16
Author: Sangjin Lee <sj...@apache.org>
Authored: Mon Mar 28 15:50:03 2016 -0700
Committer: Li Lu <gt...@apache.org>
Committed: Wed May 4 16:35:23 2016 -0700
----------------------------------------------------------------------
.../dev-support/findbugs-exclude.xml | 11 +-
.../records/timelineservice/TimelineEntity.java | 25 ++-
.../client/api/impl/TimelineClientImpl.java | 35 ++--
.../api/impl/TestTimelineClientV2Impl.java | 91 +++++++-
.../metrics/ContainerMetricsConstants.java | 8 +
.../nodemanager/NodeStatusUpdaterImpl.java | 10 +-
.../collectormanager/NMCollectorService.java | 10 +-
.../application/Application.java | 4 -
.../application/ApplicationImpl.java | 24 +--
.../timelineservice/NMTimelinePublisher.java | 210 +++++++++++--------
.../TestNMTimelinePublisher.java | 24 +--
.../yarn/server/nodemanager/webapp/MockApp.java | 5 -
12 files changed, 278 insertions(+), 179 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index ba1af88..72fbb35 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -117,8 +117,15 @@
<!-- Object cast is based on the event type -->
<Match>
- <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher$ApplicationEventHandler" />
- <Bug pattern="BC_UNCONFIRMED_CAST" />
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" />
+ <Method name="publishApplicationEvent" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher" />
+ <Method name="publishLocalizationEvent" />
+ <Bug pattern="BC_UNCONFIRMED_CAST" />
</Match>
<Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
index acc132e..7ce8279 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/timelineservice/TimelineEntity.java
@@ -17,15 +17,6 @@
*/
package org.apache.hadoop.yarn.api.records.timelineservice;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.yarn.util.TimelineServiceHelper;
-import org.codehaus.jackson.annotate.JsonSetter;
-
-import javax.xml.bind.annotation.XmlAccessType;
-import javax.xml.bind.annotation.XmlAccessorType;
-import javax.xml.bind.annotation.XmlElement;
-import javax.xml.bind.annotation.XmlRootElement;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
@@ -33,6 +24,16 @@ import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlElement;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.yarn.util.TimelineServiceHelper;
+import org.codehaus.jackson.annotate.JsonSetter;
+
/**
* The basic timeline entity data structure for timeline service v2. Timeline
* entity objects are not thread safe and should not be accessed concurrently.
@@ -564,6 +565,10 @@ public class TimelineEntity implements Comparable<TimelineEntity> {
}
public String toString() {
- return identifier.toString();
+ if (real == null) {
+ return identifier.toString();
+ } else {
+ return real.toString();
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 87a5e9c..ef8838e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -429,9 +429,8 @@ public class TimelineClientImpl extends TimelineClient {
URI uri = constructResURI(getConfig(), timelineServiceAddress, true);
putObjects(uri, path, params, obj);
needRetry = false;
- } catch (Exception e) {
- // TODO only handle exception for timelineServiceAddress being updated.
- // skip retry for other exceptions.
+ } catch (IOException e) {
+ // handle exception for timelineServiceAddress being updated.
checkRetryWithSleep(retries, e);
retries--;
}
@@ -458,29 +457,27 @@ public class TimelineClientImpl extends TimelineClient {
* @param retries
* @param e
*/
- private void checkRetryWithSleep(int retries, Exception e) throws
- YarnException, IOException {
+ private void checkRetryWithSleep(int retries, IOException e)
+ throws YarnException, IOException {
if (retries > 0) {
try {
Thread.sleep(this.serviceRetryInterval);
} catch (InterruptedException ex) {
Thread.currentThread().interrupt();
+ throw new YarnException("Interrupted while retrying to connect to ATS");
}
} else {
- LOG.error("TimelineClient has reached to max retry times :" +
- this.maxServiceRetries + " for service address: " +
- timelineServiceAddress);
- if (e instanceof YarnException) {
- throw (YarnException)e;
- } else if (e instanceof IOException) {
- throw (IOException)e;
- } else {
- throw new YarnException(e);
- }
+ StringBuilder msg =
+ new StringBuilder("TimelineClient has reached to max retry times : ");
+ msg.append(this.maxServiceRetries);
+ msg.append(" for service address: ");
+ msg.append(timelineServiceAddress);
+ LOG.error(msg.toString());
+ throw new IOException(msg.toString(), e);
}
}
- private void putObjects(
+ protected void putObjects(
URI base, String path, MultivaluedMap<String, String> params, Object obj)
throws IOException, YarnException {
ClientResponse resp;
@@ -636,17 +633,19 @@ public class TimelineClientImpl extends TimelineClient {
/**
* Poll TimelineServiceAddress for maximum of retries times if it is null.
+ *
* @param retries
* @return the left retry times
+ * @throws IOException
*/
- private int pollTimelineServiceAddress(int retries) {
+ private int pollTimelineServiceAddress(int retries) throws YarnException {
while (timelineServiceAddress == null && retries > 0) {
try {
Thread.sleep(this.serviceRetryInterval);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
+ throw new YarnException("Interrupted while trying to connect ATS");
}
- // timelineServiceAddress = getTimelineServiceAddress();
retries--;
}
return retries;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
index 7803f94..71dafdc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClientV2Impl.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.client.api.impl;
import java.io.IOException;
+import java.net.URI;
import java.util.ArrayList;
import java.util.List;
@@ -34,23 +35,33 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
public class TestTimelineClientV2Impl {
private static final Log LOG =
LogFactory.getLog(TestTimelineClientV2Impl.class);
private TestV2TimelineClient client;
private static long TIME_TO_SLEEP = 150;
+ private static final String EXCEPTION_MSG = "Exception in the content";
@Before
public void setup() {
- YarnConfiguration conf = new YarnConfiguration();
+ conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
conf.setFloat(YarnConfiguration.TIMELINE_SERVICE_VERSION, 1.0f);
conf.setInt(YarnConfiguration.NUMBER_OF_ASYNC_ENTITIES_TO_MERGE, 3);
- client = createTimelineClient(conf);
+ if (!currTestName.getMethodName()
+ .contains("testRetryOnConnectionFailure")) {
+ client = createTimelineClient(conf);
+ }
}
+ @Rule
+ public TestName currTestName = new TestName();
+ private YarnConfiguration conf;
+
private TestV2TimelineClient createTimelineClient(YarnConfiguration conf) {
ApplicationId id = ApplicationId.newInstance(0, 0);
TestV2TimelineClient client = new TestV2TimelineClient(id);
@@ -59,9 +70,34 @@ public class TestTimelineClientV2Impl {
return client;
}
- private class TestV2TimelineClient extends TimelineClientImpl {
+ private class TestV2TimelineClientForExceptionHandling
+ extends TimelineClientImpl {
+ public TestV2TimelineClientForExceptionHandling(ApplicationId id) {
+ super(id);
+ }
+
+ protected boolean throwYarnException;
+
+ public void setThrowYarnException(boolean throwYarnException) {
+ this.throwYarnException = throwYarnException;
+ }
+
+ @Override
+ protected void putObjects(URI base, String path,
+ MultivaluedMap<String, String> params, Object obj)
+ throws IOException, YarnException {
+ if (throwYarnException) {
+ throw new YarnException(EXCEPTION_MSG);
+ } else {
+ throw new IOException(
+ "Failed to get the response from the timeline server.");
+ }
+ }
+ }
+
+ private class TestV2TimelineClient
+ extends TestV2TimelineClientForExceptionHandling {
private boolean sleepBeforeReturn;
- private boolean throwException;
private List<TimelineEntities> publishedEntities;
@@ -75,10 +111,6 @@ public class TestTimelineClientV2Impl {
this.sleepBeforeReturn = sleepBeforeReturn;
}
- public void setThrowException(boolean throwException) {
- this.throwException = throwException;
- }
-
public int getNumOfTimelineEntitiesPublished() {
return publishedEntities.size();
}
@@ -91,7 +123,7 @@ public class TestTimelineClientV2Impl {
protected void putObjects(String path,
MultivaluedMap<String, String> params, Object obj)
throws IOException, YarnException {
- if (throwException) {
+ if (throwYarnException) {
throw new YarnException("ActualException");
}
publishedEntities.add((TimelineEntities) obj);
@@ -106,6 +138,45 @@ public class TestTimelineClientV2Impl {
}
@Test
+ public void testExceptionMultipleRetry() {
+ TestV2TimelineClientForExceptionHandling client =
+ new TestV2TimelineClientForExceptionHandling(
+ ApplicationId.newInstance(0, 0));
+ int maxRetries = 2;
+ conf.setInt(YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
+ maxRetries);
+ client.init(conf);
+ client.start();
+ client.setTimelineServiceAddress("localhost:12345");
+ try {
+ client.putEntities(new TimelineEntity());
+ } catch (IOException e) {
+ Assert.fail("YARN exception is expected");
+ } catch (YarnException e) {
+ Throwable cause = e.getCause();
+ Assert.assertTrue("IOException is expected",
+ cause instanceof IOException);
+ Assert.assertTrue("YARN exception is expected",
+ cause.getMessage().contains(
+ "TimelineClient has reached to max retry times : " + maxRetries));
+ }
+
+ client.setThrowYarnException(true);
+ try {
+ client.putEntities(new TimelineEntity());
+ } catch (IOException e) {
+ Assert.fail("YARN exception is expected");
+ } catch (YarnException e) {
+ Throwable cause = e.getCause();
+ Assert.assertTrue("YARN exception is expected",
+ cause instanceof YarnException);
+ Assert.assertTrue("YARN exception is expected",
+ cause.getMessage().contains(EXCEPTION_MSG));
+ }
+ client.stop();
+ }
+
+ @Test
public void testPostEntities() throws Exception {
try {
client.putEntities(generateEntity("1"));
@@ -189,7 +260,7 @@ public class TestTimelineClientV2Impl {
@Test
public void testExceptionCalls() throws Exception {
- client.setThrowException(true);
+ client.setThrowYarnException(true);
try {
client.putEntitiesAsync(generateEntity("1"));
} catch (YarnException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
index 7b42994..eadb5b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
@@ -69,4 +69,12 @@ public class ContainerMetricsConstants {
public static final String ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO =
"YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS";
+
+ // Event of this type will be emitted by NM.
+ public static final String LOCALIZATION_START_EVENT_TYPE =
+ "YARN_NM_CONTAINER_LOCALIZATION_STARTED";
+
+ // Event of this type will be emitted by NM.
+ public static final String LOCALIZATION_FINISHED_EVENT_TYPE =
+ "YARN_NM_CONTAINER_LOCALIZATION_FINISHED";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index bf07600..b618282 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -85,6 +84,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.nodelabels.NodeLabelsProvider;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerHardwareUtils;
import org.apache.hadoop.yarn.util.YarnVersionInfo;
@@ -914,9 +914,11 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
LOG.debug("Sync a new collector address: " + collectorAddr +
" for application: " + appId + " from RM.");
}
- TimelineClient client = application.getTimelineClient();
- if (client != null) {
- client.setTimelineServiceAddress(collectorAddr);
+ NMTimelinePublisher nmTimelinePublisher =
+ context.getNMTimelinePublisher();
+ if (nmTimelinePublisher != null) {
+ nmTimelinePublisher.setTimelineServiceAddress(
+ application.getAppId(), collectorAddr);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index 548c861..d667c0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
@@ -42,6 +41,7 @@ import org.apache.hadoop.yarn.server.api.records.AppCollectorsMap;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
/**
* Service that handles collector information. It is used only if the timeline
@@ -116,10 +116,10 @@ public class NMCollectorService extends CompositeService implements
String collectorAddr = collector.getCollectorAddr();
newCollectorsMap.put(appId, collectorAddr);
// set registered collector address to TimelineClient.
- TimelineClient client =
- context.getApplications().get(appId).getTimelineClient();
- if (client != null) {
- client.setTimelineServiceAddress(collectorAddr);
+ NMTimelinePublisher nmTimelinePublisher =
+ context.getNMTimelinePublisher();
+ if (nmTimelinePublisher != null) {
+ nmTimelinePublisher.setTimelineServiceAddress(appId, collectorAddr);
}
}
((NodeManager.NMContext)context).addRegisteredCollectors(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
index 5de3398..aee0862 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
@@ -22,7 +22,6 @@ import java.util.Map;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -41,7 +40,4 @@ public interface Application extends EventHandler<ApplicationEvent> {
String getFlowVersion();
long getFlowRunId();
-
- TimelineClient getTimelineClient();
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 93c6758..b9786e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -35,7 +35,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
-import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.nodemanager.Context;
@@ -50,6 +49,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.even
import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePublisher;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
@@ -76,7 +76,6 @@ public class ApplicationImpl implements Application {
private final ReadLock readLock;
private final WriteLock writeLock;
private final Context context;
- private TimelineClient timelineClient;
private static final Log LOG = LogFactory.getLog(ApplicationImpl.class);
@@ -110,7 +109,7 @@ public class ApplicationImpl implements Application {
}
this.flowContext = flowContext;
if (YarnConfiguration.systemMetricsPublisherEnabled(conf)) {
- createAndStartTimelineClient(conf);
+ context.getNMTimelinePublisher().createTimelineClient(appId);
}
}
}
@@ -142,13 +141,6 @@ public class ApplicationImpl implements Application {
}
}
- private void createAndStartTimelineClient(Configuration conf) {
- // create and start timeline client
- this.timelineClient = TimelineClient.createTimelineClient(appId);
- timelineClient.init(conf);
- timelineClient.start();
- }
-
@Override
public String getUser() {
return user.toString();
@@ -160,11 +152,6 @@ public class ApplicationImpl implements Application {
}
@Override
- public TimelineClient getTimelineClient() {
- return timelineClient;
- }
-
- @Override
public ApplicationState getApplicationState() {
this.readLock.lock();
try {
@@ -494,9 +481,10 @@ public class ApplicationImpl implements Application {
registeredCollectors.remove(app.getAppId());
}
// stop timelineClient when application get finished.
- TimelineClient timelineClient = app.getTimelineClient();
- if (timelineClient != null) {
- timelineClient.stop();
+ NMTimelinePublisher nmTimelinePublisher =
+ app.context.getNMTimelinePublisher();
+ if (nmTimelinePublisher != null) {
+ nmTimelinePublisher.stopTimelineClient(app.getAppId());
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 70b7e8d..4d3dafd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
+import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -29,7 +31,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
-import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.timelineservice.ContainerEntity;
import org.apache.hadoop.yarn.api.records.timelineservice.TimelineEntity;
@@ -41,16 +42,15 @@ import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationContainerFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl.ContainerMetric;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -72,9 +72,12 @@ public class NMTimelinePublisher extends CompositeService {
private String httpAddress;
+ protected final Map<ApplicationId, TimelineClient> appToClientMap;
+
public NMTimelinePublisher(Context context) {
super(NMTimelinePublisher.class.getName());
this.context = context;
+ appToClientMap = new ConcurrentHashMap<>();
}
@Override
@@ -82,12 +85,6 @@ public class NMTimelinePublisher extends CompositeService {
dispatcher = new AsyncDispatcher();
dispatcher.register(NMTimelineEventType.class,
new ForwardingEventHandler());
- dispatcher
- .register(ContainerEventType.class, new ContainerEventHandler());
- dispatcher.register(ApplicationEventType.class,
- new ApplicationEventHandler());
- dispatcher.register(LocalizationEventType.class,
- new LocalizationEventDispatcher());
addIfService(dispatcher);
super.serviceInit(conf);
}
@@ -112,7 +109,6 @@ public class NMTimelinePublisher extends CompositeService {
}
}
- @SuppressWarnings("unchecked")
public void reportContainerResourceUsage(Container container, Long pmemUsage,
Float cpuUsagePercentPerCore) {
if (pmemUsage != ResourceCalculatorProcessTree.UNAVAILABLE ||
@@ -133,15 +129,32 @@ public class NMTimelinePublisher extends CompositeService {
Math.round(cpuUsagePercentPerCore));
entity.addMetric(cpuMetric);
}
- dispatcher.getEventHandler()
- .handle(new TimelinePublishEvent(entity, container.getContainerId()
- .getApplicationAttemptId().getApplicationId()));
+ ApplicationId appId = container.getContainerId().getApplicationAttemptId()
+ .getApplicationId();
+ try {
+ // no need to put it as part of publisher as timeline client already has
+ // Queuing concept
+ TimelineClient timelineClient = getTimelineClient(appId);
+ if (timelineClient != null) {
+ timelineClient.putEntitiesAsync(entity);
+ } else {
+ LOG.error("Seems like client has been removed before the container"
+ + " metric could be published for " + container.getContainerId());
+ }
+ } catch (IOException | YarnException e) {
+ LOG.error("Failed to publish Container metrics for container "
+ + container.getContainerId(), e);
+ }
}
}
- private void publishContainerCreatedEvent(ContainerEntity entity,
- ContainerId containerId, Resource resource, Priority priority,
- long timestamp) {
+ @SuppressWarnings("unchecked")
+ private void publishContainerCreatedEvent(ContainerEvent event) {
+ ContainerId containerId = event.getContainerID();
+ ContainerEntity entity = createContainerEntity(containerId);
+ Container container = context.getContainers().get(containerId);
+ Resource resource = container.getResource();
+
Map<String, Object> entityInfo = new HashMap<String, Object>();
entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_ENTITY_INFO,
resource.getMemory());
@@ -152,7 +165,7 @@ public class NMTimelinePublisher extends CompositeService {
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_ENTITY_INFO,
nodeId.getPort());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
- priority.toString());
+ container.getPriority().toString());
entityInfo.put(
ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
httpAddress);
@@ -160,13 +173,15 @@ public class NMTimelinePublisher extends CompositeService {
TimelineEvent tEvent = new TimelineEvent();
tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE);
- tEvent.setTimestamp(timestamp);
+ tEvent.setTimestamp(event.getTimestamp());
entity.addEvent(tEvent);
- entity.setCreatedTime(timestamp);
- putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
+ entity.setCreatedTime(event.getTimestamp());
+ dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity,
+ containerId.getApplicationAttemptId().getApplicationId()));
}
+ @SuppressWarnings("unchecked")
private void publishContainerFinishedEvent(ContainerStatus containerStatus,
long timeStamp) {
ContainerId containerId = containerStatus.getContainerId();
@@ -186,7 +201,38 @@ public class NMTimelinePublisher extends CompositeService {
tEvent.setInfo(eventInfo);
entity.addEvent(tEvent);
- putEntity(entity, containerId.getApplicationAttemptId().getApplicationId());
+
+ dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity,
+ containerId.getApplicationAttemptId().getApplicationId()));
+ }
+
+ private void publishContainerLocalizationEvent(
+ ContainerLocalizationEvent event, String eventType) {
+ Container container = event.getContainer();
+ ContainerId containerId = container.getContainerId();
+ TimelineEntity entity = createContainerEntity(containerId);
+
+ TimelineEvent tEvent = new TimelineEvent();
+ tEvent.setId(eventType);
+ tEvent.setTimestamp(event.getTimestamp());
+ entity.addEvent(tEvent);
+
+ ApplicationId appId =
+ container.getContainerId().getApplicationAttemptId().getApplicationId();
+ try {
+ // no need to put it as part of publisher as timeline client already has
+ // Queuing concept
+ TimelineClient timelineClient = getTimelineClient(appId);
+ if (timelineClient != null) {
+ timelineClient.putEntitiesAsync(entity);
+ } else {
+ LOG.error("Seems like client has been removed before the event could be"
+ + " published for " + container.getContainerId());
+ }
+ } catch (IOException | YarnException e) {
+ LOG.error("Failed to publish Container metrics for container "
+ + container.getContainerId(), e);
+ }
}
private static ContainerEntity createContainerEntity(
@@ -207,23 +253,33 @@ public class NMTimelinePublisher extends CompositeService {
LOG.debug("Publishing the entity " + entity + ", JSON-style content: "
+ TimelineUtils.dumpTimelineRecordtoJSON(entity));
}
- TimelineClient timelineClient =
- context.getApplications().get(appId).getTimelineClient();
- timelineClient.putEntities(entity);
+ TimelineClient timelineClient = getTimelineClient(appId);
+ if (timelineClient != null) {
+ timelineClient.putEntities(entity);
+ } else {
+ LOG.error("Seems like client has been removed before the entity "
+ + "could be published for " + entity);
+ }
} catch (Exception e) {
LOG.error("Error when publishing entity " + entity, e);
}
}
- @SuppressWarnings("unchecked")
public void publishApplicationEvent(ApplicationEvent event) {
// publish only when the desired event is received
switch (event.getType()) {
case INIT_APPLICATION:
case FINISH_APPLICATION:
- case APPLICATION_CONTAINER_FINISHED:
case APPLICATION_LOG_HANDLING_FAILED:
- dispatcher.getEventHandler().handle(event);
+ // TODO need to be handled in future,
+ // not sure to publish under which entity
+ break;
+ case APPLICATION_CONTAINER_FINISHED:
+ // this is actually used to publish the container Event
+ ApplicationContainerFinishedEvent evnt =
+ (ApplicationContainerFinishedEvent) event;
+ publishContainerFinishedEvent(evnt.getContainerStatus(),
+ event.getTimestamp());
break;
default:
@@ -235,12 +291,11 @@ public class NMTimelinePublisher extends CompositeService {
}
}
- @SuppressWarnings("unchecked")
public void publishContainerEvent(ContainerEvent event) {
// publish only when the desired event is received
switch (event.getType()) {
case INIT_CONTAINER:
- dispatcher.getEventHandler().handle(event);
+ publishContainerCreatedEvent(event);
break;
default:
@@ -253,15 +308,17 @@ public class NMTimelinePublisher extends CompositeService {
}
}
- @SuppressWarnings("unchecked")
public void publishLocalizationEvent(LocalizationEvent event) {
// publish only when the desired event is received
switch (event.getType()) {
case CONTAINER_RESOURCES_LOCALIZED:
+ publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
+ ContainerMetricsConstants.LOCALIZATION_FINISHED_EVENT_TYPE);
+ break;
case INIT_CONTAINER_RESOURCES:
- dispatcher.getEventHandler().handle(event);
+ publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
+ ContainerMetricsConstants.LOCALIZATION_START_EVENT_TYPE);
break;
-
default:
if (LOG.isDebugEnabled()) {
LOG.debug(event.getType()
@@ -272,64 +329,6 @@ public class NMTimelinePublisher extends CompositeService {
}
}
- private class ApplicationEventHandler implements
- EventHandler<ApplicationEvent> {
- @Override
- public void handle(ApplicationEvent event) {
- switch (event.getType()) {
- case APPLICATION_CONTAINER_FINISHED:
- // this is actually used to publish the container Event
- ApplicationContainerFinishedEvent evnt =
- (ApplicationContainerFinishedEvent) event;
- publishContainerFinishedEvent(evnt.getContainerStatus(),
- event.getTimestamp());
- break;
- default:
- LOG.error("Seems like event type is captured only in "
- + "publishApplicationEvent method and not handled here");
- break;
- }
- }
- }
-
- private class ContainerEventHandler implements EventHandler<ContainerEvent> {
- @Override
- public void handle(ContainerEvent event) {
- ContainerId containerId = event.getContainerID();
- Container container = context.getContainers().get(containerId);
- long timestamp = event.getTimestamp();
- ContainerEntity entity = createContainerEntity(containerId);
-
- switch (event.getType()) {
- case INIT_CONTAINER:
- publishContainerCreatedEvent(entity, containerId,
- container.getResource(), container.getPriority(), timestamp);
- break;
- default:
- LOG.error("Seems like event type is captured only in "
- + "publishContainerEvent method and not handled here");
- break;
- }
- }
- }
-
- private static final class LocalizationEventDispatcher implements
- EventHandler<LocalizationEvent> {
- @Override
- public void handle(LocalizationEvent event) {
- switch (event.getType()) {
- case INIT_CONTAINER_RESOURCES:
- case CONTAINER_RESOURCES_LOCALIZED:
- // TODO after priority based flush jira is finished
- break;
- default:
- LOG.error("Seems like event type is captured only in "
- + "publishLocalizationEvent method and not handled here");
- break;
- }
- }
- }
-
/**
* EventHandler implementation which forward events to NMMetricsPublisher.
* Making use of it, NMMetricsPublisher can avoid to have a public handle
@@ -363,4 +362,33 @@ public class NMTimelinePublisher extends CompositeService {
return entityToPublish;
}
}
+
+ public void createTimelineClient(ApplicationId appId) {
+ if (!appToClientMap.containsKey(appId)) {
+ TimelineClient timelineClient =
+ TimelineClient.createTimelineClient(appId);
+ timelineClient.init(getConfig());
+ timelineClient.start();
+ appToClientMap.put(appId, timelineClient);
+ }
+ }
+
+ public void stopTimelineClient(ApplicationId appId) {
+ TimelineClient client = appToClientMap.remove(appId);
+ if (client != null) {
+ client.stop();
+ }
+ }
+
+ public void setTimelineServiceAddress(ApplicationId appId,
+ String collectorAddr) {
+ TimelineClient client = appToClientMap.get(appId);
+ if (client != null) {
+ client.setTimelineServiceAddress(collectorAddr);
+ }
+ }
+
+ private TimelineClient getTimelineClient(ApplicationId appId) {
+ return appToClientMap.get(appId);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
index 830ed6b..4aa28d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/TestNMTimelinePublisher.java
@@ -20,14 +20,12 @@ package org.apache.hadoop.yarn.server.nodemanager.timelineservice;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
-import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.IOException;
import java.util.Iterator;
import java.util.Map.Entry;
-import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -39,7 +37,6 @@ import org.apache.hadoop.yarn.api.records.timelineservice.TimelineMetric;
import org.apache.hadoop.yarn.client.api.impl.TimelineClientImpl;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.nodemanager.Context;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.util.ResourceCalculatorProcessTree;
import org.junit.Assert;
@@ -53,20 +50,23 @@ public class TestNMTimelinePublisher {
public void testContainerResourceUsage() {
Context context = mock(Context.class);
@SuppressWarnings("unchecked")
- ConcurrentMap<ApplicationId, Application> map = mock(ConcurrentMap.class);
- Application aApp = mock(Application.class);
- when(map.get(any(ApplicationId.class))).thenReturn(aApp);
- DummyTimelineClient timelineClient = new DummyTimelineClient();
- when(aApp.getTimelineClient()).thenReturn(timelineClient);
- when(context.getApplications()).thenReturn(map);
+ final DummyTimelineClient timelineClient = new DummyTimelineClient();
when(context.getNodeId()).thenReturn(NodeId.newInstance("localhost", 0));
when(context.getHttpPort()).thenReturn(0);
- NMTimelinePublisher publisher = new NMTimelinePublisher(context);
+ NMTimelinePublisher publisher = new NMTimelinePublisher(context) {
+ public void createTimelineClient(ApplicationId appId) {
+ if (!appToClientMap.containsKey(appId)) {
+ appToClientMap.put(appId, timelineClient);
+ }
+ }
+ };
publisher.init(new Configuration());
publisher.start();
+ ApplicationId appId = ApplicationId.newInstance(0, 1);
+ publisher.createTimelineClient(appId);
Container aContainer = mock(Container.class);
when(aContainer.getContainerId()).thenReturn(ContainerId.newContainerId(
- ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 1),
+ ApplicationAttemptId.newInstance(appId, 1),
0L));
publisher.reportContainerResourceUsage(aContainer, 1024L, 8F);
verifyPublishedResourceUsageMetrics(timelineClient, 1024L, 8);
@@ -141,7 +141,7 @@ public class TestNMTimelinePublisher {
private TimelineEntity[] lastPublishedEntities;
@Override
- public void putEntities(TimelineEntity... entities)
+ public void putEntitiesAsync(TimelineEntity... entities)
throws IOException, YarnException {
this.lastPublishedEntities = entities;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/899468de/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
index 4d1be84..c983040 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockApp.java
@@ -101,9 +101,4 @@ public class MockApp implements Application {
public long getFlowRunId() {
return flowRunId;
}
-
- @Override
- public TimelineClient getTimelineClient() {
- return timelineClient;
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org