You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2016/01/22 12:48:45 UTC
[1/2] falcon git commit: FALCON-1757 EntityNotRegisteredException
when entity is deleted from falcon (By Pavan Kolamuri)
Repository: falcon
Updated Branches:
refs/heads/master 65ee6106b -> a86ddb0bd
FALCON-1757 EntityNotRegisteredException when entity is deleted from falcon (By Pavan Kolamuri)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/32f1da60
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/32f1da60
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/32f1da60
Branch: refs/heads/master
Commit: 32f1da60c410e1605b791dc34e10b2a0ef0df11a
Parents: 65ee610
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Fri Jan 22 16:24:41 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Fri Jan 22 16:24:41 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../falcon/rerun/handler/AbstractRerunConsumer.java | 9 ++++++++-
.../apache/falcon/rerun/handler/LateRerunConsumer.java | 9 ++++++++-
.../org/apache/falcon/rerun/handler/LateRerunHandler.java | 7 +++++++
.../org/apache/falcon/rerun/handler/RetryConsumer.java | 10 ++++++++--
5 files changed, 33 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/32f1da60/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 74da042..23c8b99 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -120,6 +120,8 @@ Proposed Release Version: 0.9
OPTIMIZATIONS
BUG FIXES
+ FALCON-1757 EntityNotRegisteredException when entity is deleted from falcon (Pavan Kolamuri via Pallavi Rao)
+
FALCON-1748 Client throws FalconWebException irrespective of type of error(Praveen Adlakha via Ajay Yadava)
FALCON-1727 Suspend fails with InvalidStateTransitionException if entity has 'KILLED' instances (Pallavi Rao)
http://git-wip-us.apache.org/repos/asf/falcon/blob/32f1da60/rerun/src/main/java/org/apache/falcon/rerun/handler/AbstractRerunConsumer.java
----------------------------------------------------------------------
diff --git a/rerun/src/main/java/org/apache/falcon/rerun/handler/AbstractRerunConsumer.java b/rerun/src/main/java/org/apache/falcon/rerun/handler/AbstractRerunConsumer.java
index f60b927..61ca8c0 100644
--- a/rerun/src/main/java/org/apache/falcon/rerun/handler/AbstractRerunConsumer.java
+++ b/rerun/src/main/java/org/apache/falcon/rerun/handler/AbstractRerunConsumer.java
@@ -20,6 +20,7 @@ package org.apache.falcon.rerun.handler;
import org.apache.commons.lang3.exception.ExceptionUtils;
import org.apache.falcon.FalconException;
import org.apache.falcon.aspect.GenericAlert;
+import org.apache.falcon.entity.EntityNotRegisteredException;
import org.apache.falcon.entity.v0.Frequency;
import org.apache.falcon.rerun.event.RerunEvent;
import org.apache.falcon.rerun.policy.AbstractRerunPolicy;
@@ -53,8 +54,8 @@ public abstract class AbstractRerunConsumer<T extends RerunEvent, M extends Abst
AbstractRerunPolicy policy = new ExpBackoffPolicy();
Frequency frequency = new Frequency("minutes(1)");
while (!Thread.currentThread().isInterrupted()) {
+ T message = null;
try {
- T message;
try {
message = handler.takeFromQueue();
attempt = 1;
@@ -83,6 +84,12 @@ public abstract class AbstractRerunConsumer<T extends RerunEvent, M extends Abst
message.getEntityType(), message.getEntityName());
} catch (Throwable e) {
+ if (e instanceof EntityNotRegisteredException) {
+ LOG.warn("Entity {} of type {} doesn't exist in config store. Rerun "
+ + "cannot be done for workflow ", message.getEntityName(),
+ message.getEntityType(), message.getWfId());
+ return;
+ }
LOG.error("Error in rerun consumer", e);
}
}
http://git-wip-us.apache.org/repos/asf/falcon/blob/32f1da60/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunConsumer.java
----------------------------------------------------------------------
diff --git a/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunConsumer.java b/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunConsumer.java
index 4297788..fa0d6ae 100644
--- a/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunConsumer.java
+++ b/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunConsumer.java
@@ -18,6 +18,7 @@
package org.apache.falcon.rerun.handler;
import org.apache.falcon.aspect.GenericAlert;
+import org.apache.falcon.entity.EntityNotRegisteredException;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.SchemaHelper;
@@ -58,7 +59,7 @@ public class LateRerunConsumer<T extends LateRerunHandler<DelayedQueue<LaterunEv
if (jobStatus.equals("RUNNING") || jobStatus.equals("PREP")
|| jobStatus.equals("SUSPENDED")) {
LOG.debug("Re-enqueing message in LateRerunHandler for workflow with same delay as "
- + "job status is running: {}", message.getWfId());
+ + "job status is running: {}", message.getWfId());
message.setMsgInsertTime(System.currentTimeMillis());
handler.offerToQueue(message);
return;
@@ -81,6 +82,12 @@ public class LateRerunConsumer<T extends LateRerunHandler<DelayedQueue<LaterunEv
LOG.info("Scheduled late rerun for wf-id: {} on cluster: {}",
message.getWfId(), message.getClusterName());
} catch (Exception e) {
+ if (e instanceof EntityNotRegisteredException) {
+ LOG.warn("Entity {} of type {} doesn't exist in config store. Late rerun "
+ + "cannot be done for workflow ", message.getEntityName(),
+ message.getEntityType(), message.getWfId());
+ return;
+ }
LOG.warn("Late Re-run failed for instance {}:{} after {}",
message.getEntityName(), message.getInstance(), message.getDelayInMilliSec(), e);
GenericAlert.alertLateRerunFailed(message.getEntityType(), message.getEntityName(),
http://git-wip-us.apache.org/repos/asf/falcon/blob/32f1da60/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunHandler.java
----------------------------------------------------------------------
diff --git a/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunHandler.java b/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunHandler.java
index 1d2ed37..0be6252 100644
--- a/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunHandler.java
+++ b/rerun/src/main/java/org/apache/falcon/rerun/handler/LateRerunHandler.java
@@ -20,6 +20,7 @@ package org.apache.falcon.rerun.handler;
import org.apache.falcon.FalconException;
import org.apache.falcon.aspect.GenericAlert;
+import org.apache.falcon.entity.EntityNotRegisteredException;
import org.apache.falcon.entity.EntityUtil;
import org.apache.falcon.entity.store.ConfigurationStore;
import org.apache.falcon.entity.v0.Entity;
@@ -91,6 +92,12 @@ public class LateRerunHandler<M extends DelayedQueue<LaterunEvent>> extends
wait, entityType, entityName, nominalTime, intRunId, workflowUser);
offerToQueue(event);
} catch (Exception e) {
+ if (e instanceof EntityNotRegisteredException) {
+ LOG.warn("Entity {} of type {} doesn't exist in config store. So late rerun "
+ + "cannot be done for workflow ", entityName,
+ entityType, wfId);
+ return;
+ }
LOG.error("Unable to schedule late rerun for entity instance: {} ({}): {} And WorkflowId: {}",
entityType, entityName, nominalTime, wfId, e);
GenericAlert.alertLateRerunFailed(entityType, entityName,
http://git-wip-us.apache.org/repos/asf/falcon/blob/32f1da60/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryConsumer.java
----------------------------------------------------------------------
diff --git a/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryConsumer.java b/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryConsumer.java
index 96300d9..9b46713 100644
--- a/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryConsumer.java
+++ b/rerun/src/main/java/org/apache/falcon/rerun/handler/RetryConsumer.java
@@ -18,6 +18,7 @@
package org.apache.falcon.rerun.handler;
import org.apache.falcon.aspect.GenericAlert;
+import org.apache.falcon.entity.EntityNotRegisteredException;
import org.apache.falcon.entity.v0.SchemaHelper;
import org.apache.falcon.rerun.event.RetryEvent;
import org.apache.falcon.rerun.queue.DelayedQueue;
@@ -43,17 +44,22 @@ public class RetryConsumer<T extends RetryHandler<DelayedQueue<RetryEvent>>>
try {
if (!jobStatus.equals("KILLED")) {
LOG.debug("Re-enqueing message in RetryHandler for workflow with same delay as job status is running:"
- + " {}", message.getWfId());
+ + " {}", message.getWfId());
message.setMsgInsertTime(System.currentTimeMillis());
handler.offerToQueue(message);
return;
}
LOG.info("Retrying attempt: {} out of configured: {} attempt for instance: {}:{} And WorkflowId: {}"
- + " At time: {}",
+ + " At time: {}",
(message.getRunId() + 1), message.getAttempts(), message.getEntityName(), message.getInstance(),
message.getWfId(), SchemaHelper.formatDateUTC(new Date(System.currentTimeMillis())));
handler.getWfEngine(entityType, entityName).reRun(message.getClusterName(), message.getWfId(), null, false);
} catch (Exception e) {
+ if (e instanceof EntityNotRegisteredException) {
+ LOG.warn("Entity {} of type {} doesn't exist in config store. So retry "
+ + "cannot be done for workflow ", entityName, entityType, message.getWfId());
+ return;
+ }
int maxFailRetryCount = Integer.parseInt(StartupProperties.get()
.getProperty("max.retry.failure.count", "1"));
if (message.getFailRetryCount() < maxFailRetryCount) {
[2/2] falcon git commit: FALCON-1716 API fails with
CommunicationsException when mysql interaction time is longer than 53, 434,
939 milliseconds (By Pavan Kolamuri)
Posted by pa...@apache.org.
FALCON-1716 API fails with CommunicationsException when mysql interaction time is longer than 53,434,939 milliseconds (By Pavan Kolamuri)
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/a86ddb0b
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/a86ddb0b
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/a86ddb0b
Branch: refs/heads/master
Commit: a86ddb0bd1c0f0e21ace590cc65f27f673cb7875
Parents: 32f1da6
Author: Pallavi Rao <pa...@inmobi.com>
Authored: Fri Jan 22 16:31:54 2016 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Fri Jan 22 16:31:54 2016 +0530
----------------------------------------------------------------------
CHANGES.txt | 2 ++
.../org/apache/falcon/state/store/service/FalconJPAService.java | 2 +-
src/conf/statestore.properties | 3 ++-
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/a86ddb0b/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index 23c8b99..54ab9b7 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -120,6 +120,8 @@ Proposed Release Version: 0.9
OPTIMIZATIONS
BUG FIXES
+ FALCON-1716 API fails with CommunicationsException when mysql interaction time is longer than 53,434,939 milliseconds (Pavan Kolamuri via Pallavi Rao)
+
FALCON-1757 EntityNotRegisteredException when entity is deleted from falcon (Pavan Kolamuri via Pallavi Rao)
FALCON-1748 Client throws FalconWebException irrespective of type of error(Praveen Adlakha via Ajay Yadava)
http://git-wip-us.apache.org/repos/asf/falcon/blob/a86ddb0b/scheduler/src/main/java/org/apache/falcon/state/store/service/FalconJPAService.java
----------------------------------------------------------------------
diff --git a/scheduler/src/main/java/org/apache/falcon/state/store/service/FalconJPAService.java b/scheduler/src/main/java/org/apache/falcon/state/store/service/FalconJPAService.java
index f678a6f..027a8ef 100644
--- a/scheduler/src/main/java/org/apache/falcon/state/store/service/FalconJPAService.java
+++ b/scheduler/src/main/java/org/apache/falcon/state/store/service/FalconJPAService.java
@@ -139,7 +139,7 @@ public final class FalconJPAService implements FalconService {
String interval = "timeBetweenEvictionRunsMillis=" + evictionInterval;
String num = "numTestsPerEvictionRun=" + evictionNum;
connProps += ",TestOnBorrow=true,TestOnReturn=true,TestWhileIdle=true," + interval + "," + num;
- connProps += ",ValidationQuery=select count(*) from ENTITIES";
+ connProps += ",ValidationQuery=select 1";
connProps = MessageFormat.format(connProps, dbSchema);
} else {
connProps += ",TestOnBorrow=false,TestOnReturn=false,TestWhileIdle=false";
http://git-wip-us.apache.org/repos/asf/falcon/blob/a86ddb0b/src/conf/statestore.properties
----------------------------------------------------------------------
diff --git a/src/conf/statestore.properties b/src/conf/statestore.properties
index 0c912ad..de5ad8f 100644
--- a/src/conf/statestore.properties
+++ b/src/conf/statestore.properties
@@ -42,4 +42,5 @@
## Creates Falcon DB.
## If set to true, it creates the DB schema if it does not exist. If the DB schema exists is a NOP.
## If set to false, it does not create the DB schema. If the DB schema does not exist it fails start up.
-#*.falcon.statestore.create.db.schema=true
\ No newline at end of file
+## In production environments it is recommended not to change this property,create schema using tool provided by falcon.
+#*.falcon.statestore.create.db.schema=false
\ No newline at end of file