You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/10/20 08:13:43 UTC
falcon git commit: FALCON-2147 Excess logging due to SlaMonitoring service and Backlog e…
Repository: falcon
Updated Branches:
refs/heads/master ef57d29ad -> 4746e039a
FALCON-2147 Excess logging due to SlaMonitoring service and Backlog e\u2026
\u2026mitter service
Author: Pallavi Rao <pa...@inmobi.com>
Reviewers: @sandeepSamudrala
Closes #286 from pallavi-rao/2147 and squashes the following commits:
b3cc9d0 [Pallavi Rao] Comments addressed
7069e2f [Pallavi Rao] Removing excessive user logging in in BacklogMetricEmitter
cacdc1d [Pallavi Rao] Addressed comment
4ddd4dd [Pallavi Rao] FALCON-2147 Excess logging due to SlaMonitoring service and Backlog emitter service
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/4746e039
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/4746e039
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/4746e039
Branch: refs/heads/master
Commit: 4746e039a1522af375a4e877848a6274b2de1c66
Parents: ef57d29
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Thu Oct 20 13:43:24 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Thu Oct 20 13:43:24 2016 +0530
----------------------------------------------------------------------
.../falcon/workflow/WorkflowEngineFactory.java | 4 ++--
.../workflow/engine/OozieWorkflowEngine.java | 2 +-
.../service/BacklogMetricEmitterService.java | 19 +++++++++++++------
.../falcon/service/EntitySLAAlertService.java | 5 ++---
.../service/EntitySLAMonitoringService.java | 19 +++++++++----------
5 files changed, 27 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java b/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
index c713712..e5d0da2 100644
--- a/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
+++ b/common/src/main/java/org/apache/falcon/workflow/WorkflowEngineFactory.java
@@ -56,7 +56,7 @@ public final class WorkflowEngineFactory {
LOG.debug("Returning native workflow engine for entity {}", entity.getName());
return nativeWorkflowEngine;
}
- LOG.debug("Returning configured workflow engine for entity {}.", entity);
+ LOG.debug("Returning configured workflow engine for entity {}", (entity == null)? null : entity.getName());
return getWorkflowEngine();
}
@@ -70,7 +70,7 @@ public final class WorkflowEngineFactory {
throws FalconException {
// If entity is null or not schedulable and the engine property is not specified, return the configured WE.
if (entity == null || !entity.getEntityType().isSchedulable()) {
- LOG.debug("Returning configured workflow engine for entity {}.", entity);
+ LOG.debug("Returning configured workflow engine for entity {}", (entity == null)? null : entity.getName());
return getWorkflowEngine();
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
index 394600c..6964200 100644
--- a/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
+++ b/oozie/src/main/java/org/apache/falcon/workflow/engine/OozieWorkflowEngine.java
@@ -371,7 +371,7 @@ public class OozieWorkflowEngine extends AbstractWorkflowEngine {
//Load bundle as coord info is not returned in getBundleJobsInfo()
BundleJob bundle = getBundleInfo(clusterName, job.getId());
filteredJobs.add(bundle);
- LOG.debug("Found bundle {} with app path {} and status {}",
+ LOG.trace("Found bundle {} with app path {} and status {}",
job.getId(), job.getAppPath(), job.getStatus());
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
index 801ab36..d9ac386 100644
--- a/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
+++ b/prism/src/main/java/org/apache/falcon/service/BacklogMetricEmitterService.java
@@ -17,6 +17,7 @@
*/
package org.apache.falcon.service;
+import org.apache.commons.lang3.StringUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.LifeCycle;
import org.apache.falcon.entity.EntityUtil;
@@ -204,7 +205,7 @@ public final class BacklogMetricEmitterService implements FalconService,
@Override
public void run() {
- LOG.debug("BacklogMetricEmitter running for entities");
+ LOG.debug("Starting periodic check for backlog");
executor = new ScheduledThreadPoolExecutor(10);
List<Future> futures = new ArrayList<>();
try {
@@ -316,11 +317,7 @@ public final class BacklogMetricEmitterService implements FalconService,
Date nominalTime;
try {
nominalTime = DATE_FORMAT.get().parse(nominalTimeStr);
- if (entity.getACL().getOwner() != null && !entity.getACL().getOwner().isEmpty()) {
- CurrentUser.authenticate(entity.getACL().getOwner());
- } else {
- CurrentUser.authenticate(System.getProperty("user.name"));
- }
+ authenticateUser(entity);
if (wfEngine.isMissing(entity)) {
LOG.info("Entity of name {} was deleted so removing instance of "
+ "nominaltime {} ", entity.getName(), nominalTimeStr);
@@ -353,4 +350,14 @@ public final class BacklogMetricEmitterService implements FalconService,
}
}
+ private static void authenticateUser(Entity entity){
+ if (!CurrentUser.isAuthenticated()) {
+ if (StringUtils.isNotBlank(entity.getACL().getOwner())) {
+ CurrentUser.authenticate(entity.getACL().getOwner());
+ } else {
+ CurrentUser.authenticate(System.getProperty("user.name"));
+ }
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java b/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
index 9b1a594..09c6695 100644
--- a/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
+++ b/prism/src/main/java/org/apache/falcon/service/EntitySLAAlertService.java
@@ -76,6 +76,7 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList
continue;
}
EntitySLAListener listener = ReflectionUtils.getInstanceByClassName(listenerClassName);
+ LOG.info("Registering listener {}" , listenerClassName);
registerListener(listener);
}
}
@@ -111,8 +112,7 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList
if (pendingInstanceBeanList == null || pendingInstanceBeanList.isEmpty()){
return;
}
-
- LOG.debug("In processSLACandidates :" + pendingInstanceBeanList.size());
+ LOG.trace("In processSLACandidates :" + pendingInstanceBeanList.size());
try{
for (PendingInstanceBean pendingInstanceBean : pendingInstanceBeanList) {
@@ -161,7 +161,6 @@ public final class EntitySLAAlertService implements FalconService, EntitySLAList
@Override
public void highSLAMissed(String entityName, String clusterName, EntityType entityType , Date nominalTime
) throws FalconException {
- LOG.debug("Listners called...");
for (EntitySLAListener listener : listeners) {
listener.highSLAMissed(entityName, clusterName, entityType, nominalTime);
store.deleteEntityAlertInstance(entityName, clusterName, nominalTime, entityType.name());
http://git-wip-us.apache.org/repos/asf/falcon/blob/4746e039/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
----------------------------------------------------------------------
diff --git a/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java b/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
index 1e20a2b..816846d 100644
--- a/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
+++ b/prism/src/main/java/org/apache/falcon/service/EntitySLAMonitoringService.java
@@ -303,14 +303,14 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
freq = StartupProperties.get().getProperty("entity.sla.lookAheadWindow.millis", "900000");
lookAheadWindowMillis = Integer.parseInt(freq);
- LOG.debug("No old state exists at: {}, Initializing a clean state.", filePath.toString());
+ LOG.info("Initializing EntitySLAMonitoringService from ", filePath.toString());
ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(1);
executor.scheduleWithFixedDelay(new Monitor(), 0, statusCheckFrequencySeconds, TimeUnit.SECONDS);
}
public void makeFeedInstanceAvailable(String feedName, String clusterName, Date nominalTime)
throws FalconException {
- LOG.info("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName,
+ LOG.debug("Removing {} feed's instance {} in cluster {} from pendingSLA", feedName,
clusterName, nominalTime);
List<Date> instances = (MONITORING_JDBC_STATE_STORE.getNominalInstances(feedName, clusterName,
EntityType.FEED.toString()));
@@ -369,7 +369,6 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
for(MonitoredEntityBean monitoredEntityBean : entityBeanList) {
String entityName = monitoredEntityBean.getFeedName();
Entity entity = EntityUtil.getEntity(entityType, entityName);
- LOG.debug("entityName:"+ entityName+"entity:"+entity);
Set<String> clusters = EntityUtil.getClustersDefined(entity);
List<org.apache.falcon.entity.v0.cluster.Cluster> cluster = new ArrayList();
for(String string : clusters){
@@ -391,7 +390,7 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
org.apache.falcon.entity.v0.cluster.Cluster currentCluster =
EntityUtil.getEntity(EntityType.CLUSTER, entityCluster.getName());
nextInstanceTime = EntityUtil.getNextStartTime(entity, currentCluster, nextInstanceTime);
- LOG.info("nextInstanceTime:"+ nextInstanceTime + "entityName:"+entityName);
+ LOG.trace("nextInstanceTime:"+ nextInstanceTime + "entityName:"+entityName);
Date endDate;
if (entityType.equals(EntityType.FEED.toString())){
endDate = FeedHelper.getClusterValidity((Feed) entity, currentCluster.getName()).getEnd();
@@ -400,7 +399,8 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
currentCluster.getName()).getEnd();
}
while (nextInstanceTime.before(to) && nextInstanceTime.before(endDate)) {
- LOG.info("Adding instance={} for <entity,cluster>={}", nextInstanceTime, key);
+ LOG.trace("Adding pending instance={} for <entity,cluster>={}; entityType={}",
+ nextInstanceTime, key, entityType);
instances.add(nextInstanceTime);
nextInstanceTime = new Date(nextInstanceTime.getTime() + ONE_MS);
nextInstanceTime = EntityUtil.getNextStartTime(entity, currentCluster, nextInstanceTime);
@@ -420,7 +420,6 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
* Checks the availability of all the pendingInstances and removes the ones which have become available.
*/
private void checkPendingInstanceAvailability(String entityType) throws FalconException {
- LOG.debug("Size "+MONITORING_JDBC_STATE_STORE.getAllMonitoredEntity().size());
if (MONITORING_JDBC_STATE_STORE.getAllPendingInstances() == null){
LOG.info("Returning as size of pending instance is zero");
return;
@@ -445,26 +444,26 @@ public final class EntitySLAMonitoringService implements ConfigurationChangeList
authenticateUser(entity);
try {
if (entityType.equals(EntityType.PROCESS.toString())){
- LOG.debug("Checking instance availability status for entity:{}, cluster:{}, "
+ LOG.trace("Checking instance availability status for entity:{}, cluster:{}, "
+ "instanceTime:{}", entity.getName(), clusterName, nominalTime, entityType);
AbstractWorkflowEngine wfEngine = WorkflowEngineFactory.getWorkflowEngine();
InstancesResult instancesResult = wfEngine.getStatus(entity, nominalTime, nominalTime, null, null);
if (instancesResult.getStatus().equals(APIResult.Status.SUCCEEDED)){
- LOG.debug("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is available.",
+ LOG.trace("Entity instance(Process:{}, cluster:{}, instanceTime:{}) is available.",
entity.getName(), clusterName, nominalTime);
return true;
}
return false;
}
if (entityType.equals(EntityType.FEED.toString())){
- LOG.debug("Checking instance availability status for feed:{}, cluster:{}, instanceTime:{}",
+ LOG.trace("Checking instance availability status for feed:{}, cluster:{}, instanceTime:{}",
entity.getName(), clusterName, nominalTime);
FeedInstanceStatus.AvailabilityStatus status = FeedHelper.getFeedInstanceStatus((Feed) entity,
clusterName, nominalTime);
if (status.equals(FeedInstanceStatus.AvailabilityStatus.AVAILABLE)
|| status.equals(FeedInstanceStatus.AvailabilityStatus.EMPTY)) {
- LOG.debug("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.", entity.getName(),
+ LOG.trace("Feed instance(feed:{}, cluster:{}, instanceTime:{}) is available.", entity.getName(),
clusterName, nominalTime);
return true;
}