You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@falcon.apache.org by "Venkatesh Seetharam (JIRA)" <ji...@apache.org> on 2014/09/17 19:12:34 UTC
[jira] [Commented] (FALCON-731) Lineage capture for evicted
instance is broken
[ https://issues.apache.org/jira/browse/FALCON-731?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14137570#comment-14137570 ]
Venkatesh Seetharam commented on FALCON-731:
--------------------------------------------
I think the assumptions for capturing lineage for eviction is completely broken with bad assumptions. The message is sent for each evicted instance in org.apache.falcon.messaging.JMSMessageProducer#buildMessageList which builds a list with each of the instance. The lineage for eviction code looks for the file with paths on hdfs which is deleted post message send. I also did not notice this during review.
> Lineage capture for evicted instance is broken
> ----------------------------------------------
>
> Key: FALCON-731
> URL: https://issues.apache.org/jira/browse/FALCON-731
> Project: Falcon
> Issue Type: Bug
> Affects Versions: 0.6
> Reporter: Venkatesh Seetharam
> Assignee: Sowmya Ramesh
> Priority: Blocker
> Fix For: 0.6
>
>
> {code}
> java.lang.IllegalStateException: No user logged into the system
> at org.apache.falcon.security.CurrentUser.getUser(CurrentUser.java:73)
> at org.apache.falcon.security.CurrentUser.getProxyUgi(CurrentUser.java:102)
> at org.apache.falcon.hadoop.HadoopClientFactory.createProxiedFileSystem(HadoopClientFactory.java:117)
> at org.apache.falcon.entity.ClusterHelper.getFileSystem(ClusterHelper.java:46)
> at org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:222)
> at org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:298)
> at org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:273)
> at org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
> at org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:131)
> at org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:103)
> at org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
> at org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
> at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
> at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
> at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:679)
> {code}
> {code}
> org.apache.falcon.FalconException: java.io.FileNotFoundException: File does not exist: /apps/falcon/sandbox/bcp/staging/falcon/workflows/feed/clicks-feed/logs/instancePaths-2014-09-16-18-45.csv
> at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:61)
> at org.apache.hadoop.hdfs.server.namenode.INodeFile.valueOf(INodeFile.java:51)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsUpdateTimes(FSNamesystem.java:1540)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocationsInt(FSNamesystem.java:1483)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1463)
> at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.getBlockLocations(FSNamesystem.java:1437)
> at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.getBlockLocations(NameNodeRpcServer.java:468)
> at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.getBlockLocations(ClientNamenodeProtocolServerSideTranslatorPB.java:269)
> at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java:59566)
> at org.apache.hadoop.ipc.ProtobufRpcEngine$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine.java:585)
> at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:928)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2053)
> at org.apache.hadoop.ipc.Server$Handler$1.run(Server.java:2049)
> at java.security.AccessController.doPrivileged(Native Method)
> at javax.security.auth.Subject.doAs(Subject.java:396)
> at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1491)
> at org.apache.hadoop.ipc.Server$Handler.run(Server.java:2047)
> at org.apache.falcon.retention.EvictionHelper.getInstancePaths(EvictionHelper.java:65)
> at org.apache.falcon.metadata.InstanceRelationshipGraphBuilder.addEvictedInstance(InstanceRelationshipGraphBuilder.java:222)
> at org.apache.falcon.metadata.MetadataMappingService.onFeedInstanceEvicted(MetadataMappingService.java:298)
> at org.apache.falcon.metadata.MetadataMappingService.onSuccess(MetadataMappingService.java:273)
> at org.apache.falcon.workflow.WorkflowJobEndNotificationService.notifySuccess(WorkflowJobEndNotificationService.java:101)
> at org.apache.falcon.messaging.JMSMessageConsumer.onSuccess(JMSMessageConsumer.java:131)
> at org.apache.falcon.messaging.JMSMessageConsumer.onMessage(JMSMessageConsumer.java:103)
> at org.apache.activemq.ActiveMQMessageConsumer.dispatch(ActiveMQMessageConsumer.java:1229)
> at org.apache.activemq.ActiveMQSessionExecutor.dispatch(ActiveMQSessionExecutor.java:134)
> at org.apache.activemq.ActiveMQSessionExecutor.iterate(ActiveMQSessionExecutor.java:205)
> at org.apache.activemq.thread.PooledTaskRunner.runTask(PooledTaskRunner.java:122)
> at org.apache.activemq.thread.PooledTaskRunner$1.run(PooledTaskRunner.java:43)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1146)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:615)
> at java.lang.Thread.run(Thread.java:679)
> {code}
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)