You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by "Mark Payne (Jira)" <ji...@apache.org> on 2020/10/23 17:55:00 UTC
[jira] [Comment Edited] (NIFI-7944) When importing a flow from
registry, Controller Service may lose track of Referencing Components
[ https://issues.apache.org/jira/browse/NIFI-7944?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17219853#comment-17219853 ]
Mark Payne edited comment on NIFI-7944 at 10/23/20, 5:54 PM:
-------------------------------------------------------------
Furthermore, it looks like if a Controller Service is referencing another Controller Service, and that referenced service got disabled and deleted, NiFi will fail to restart.
To replicate, create a Process Group, Group A. Within it, create nothing but a child group, Group B.
Within Group B, create an AvroSchemaRegistry controller service, a CSV Reader, and JSON Writer. Configure CSV Reader to use the Avro Schema Registry to get the schema.
Create GenerateFlowFile -> ConvertRecord (using CSV Reader and JSON Writer) -> (terminate)
Persist this flow to the registry. Then import the flow a second time.
On the newly imported flow, start all Controller Services, then start the Processors.
Disable AvroSchemaRegistry service.
Delete AvroSchemaRegistry service.
Restart NiFi.
It will fail to startup with an error like the following:
{code:java}
2020-10-23 13:50:27,129 WARN [main] org.apache.nifi.web.server.JettyServer Failed to start web server... shutting down.
org.apache.nifi.controller.serialization.FlowSynchronizationException: java.lang.NullPointerException
at org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:306)
at org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1420)
at org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:89)
at org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:792)
at org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:537)
at org.apache.nifi.web.contextlistener.ApplicationStartupContextListener.contextInitialized(ApplicationStartupContextListener.java:72)
at org.eclipse.jetty.server.handler.ContextHandler.callContextInitialized(ContextHandler.java:930)
at org.eclipse.jetty.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:553)
at org.eclipse.jetty.server.handler.ContextHandler.startContext(ContextHandler.java:889)
at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:356)
at org.eclipse.jetty.webapp.WebAppContext.startWebapp(WebAppContext.java:1445)
at org.eclipse.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1409)
at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:822)
at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:275)
at org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:524)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:110)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:425)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169)
at org.eclipse.jetty.server.Server.start(Server.java:407)
at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:110)
at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97)
at org.eclipse.jetty.server.Server.doStart(Server.java:371)
at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72)
at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:1054)
at org.apache.nifi.NiFi.<init>(NiFi.java:159)
at org.apache.nifi.NiFi.<init>(NiFi.java:71)
at org.apache.nifi.NiFi.main(NiFi.java:302)
Caused by: java.lang.NullPointerException: null
at org.apache.nifi.controller.service.StandardControllerServiceProvider.enableControllerServices(StandardControllerServiceProvider.java:192)
at org.apache.nifi.controller.service.ControllerServiceLoader.enableControllerServices(ControllerServiceLoader.java:164)
at org.apache.nifi.controller.service.ControllerServiceLoader.enableControllerServices(ControllerServiceLoader.java:152)
at org.apache.nifi.controller.StandardFlowSynchronizer.addControllerServices(StandardFlowSynchronizer.java:1374)
at org.apache.nifi.controller.StandardFlowSynchronizer.addProcessGroup(StandardFlowSynchronizer.java:1316)
at org.apache.nifi.controller.StandardFlowSynchronizer.addNestedProcessGroups(StandardFlowSynchronizer.java:1333)
at org.apache.nifi.controller.StandardFlowSynchronizer.addProcessGroup(StandardFlowSynchronizer.java:1322)
at org.apache.nifi.controller.StandardFlowSynchronizer.addNestedProcessGroups(StandardFlowSynchronizer.java:1333)
at org.apache.nifi.controller.StandardFlowSynchronizer.addProcessGroup(StandardFlowSynchronizer.java:1322)
at org.apache.nifi.controller.StandardFlowSynchronizer.updateFlow(StandardFlowSynchronizer.java:432)
at org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:283)
... 42 common frames omitted
{code}
was (Author: markap14):
Furthermore, it looks like if a Controller Service is referencing another Controller Service, and that referenced service got disabled and deleted, NiFi will fail to restart.
To replicate, create a Process Group, Group A. Within it, create nothing but a child group, Group B.
Within Group B, create an AvroSchemaRegistry controller service, a CSV Reader, and JSON Writer. Configure CSV Reader to use the Avro Schema Registry to get the schema.
Create GenerateFlowFile -> ConvertRecord (using CSV Reader and JSON Writer) -> (terminate)
Persist this flow to the registry. Then import the flow a second time.
On the newly imported flow, start all Controller Services, then start the Processors.
Disable AvroSchemaRegistry service.
Delete AvroSchemaRegistry service.
Restart NiFi.
It will fail to startup with an error like the following:
{code:java}
2020-10-23 13:50:27,129 WARN [main] org.apache.nifi.web.server.JettyServer Failed to start web server... shutting down.2020-10-23 13:50:27,129 WARN [main] org.apache.nifi.web.server.JettyServer Failed to start web server... shutting down.org.apache.nifi.controller.serialization.FlowSynchronizationException: java.lang.NullPointerException at org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:306) at org.apache.nifi.controller.FlowController.synchronize(FlowController.java:1420) at org.apache.nifi.persistence.StandardXMLFlowConfigurationDAO.load(StandardXMLFlowConfigurationDAO.java:89) at org.apache.nifi.controller.StandardFlowService.loadFromBytes(StandardFlowService.java:792) at org.apache.nifi.controller.StandardFlowService.load(StandardFlowService.java:537) at org.apache.nifi.web.contextlistener.ApplicationStartupContextListener.contextInitialized(ApplicationStartupContextListener.java:72) at org.eclipse.jetty.server.handler.ContextHandler.callContextInitialized(ContextHandler.java:930) at org.eclipse.jetty.servlet.ServletContextHandler.callContextInitialized(ServletContextHandler.java:553) at org.eclipse.jetty.server.handler.ContextHandler.startContext(ContextHandler.java:889) at org.eclipse.jetty.servlet.ServletContextHandler.startContext(ServletContextHandler.java:356) at org.eclipse.jetty.webapp.WebAppContext.startWebapp(WebAppContext.java:1445) at org.eclipse.jetty.webapp.WebAppContext.startContext(WebAppContext.java:1409) at org.eclipse.jetty.server.handler.ContextHandler.doStart(ContextHandler.java:822) at org.eclipse.jetty.servlet.ServletContextHandler.doStart(ServletContextHandler.java:275) at org.eclipse.jetty.webapp.WebAppContext.doStart(WebAppContext.java:524) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117) at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:110) at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97) at org.eclipse.jetty.server.handler.gzip.GzipHandler.doStart(GzipHandler.java:425) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117) at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:117) at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.eclipse.jetty.util.component.ContainerLifeCycle.start(ContainerLifeCycle.java:169) at org.eclipse.jetty.server.Server.start(Server.java:407) at org.eclipse.jetty.util.component.ContainerLifeCycle.doStart(ContainerLifeCycle.java:110) at org.eclipse.jetty.server.handler.AbstractHandler.doStart(AbstractHandler.java:97) at org.eclipse.jetty.server.Server.doStart(Server.java:371) at org.eclipse.jetty.util.component.AbstractLifeCycle.start(AbstractLifeCycle.java:72) at org.apache.nifi.web.server.JettyServer.start(JettyServer.java:1054) at org.apache.nifi.NiFi.<init>(NiFi.java:159) at org.apache.nifi.NiFi.<init>(NiFi.java:71) at org.apache.nifi.NiFi.main(NiFi.java:302)Caused by: java.lang.NullPointerException: null at org.apache.nifi.controller.service.StandardControllerServiceProvider.enableControllerServices(StandardControllerServiceProvider.java:192) at org.apache.nifi.controller.service.ControllerServiceLoader.enableControllerServices(ControllerServiceLoader.java:164) at org.apache.nifi.controller.service.ControllerServiceLoader.enableControllerServices(ControllerServiceLoader.java:152) at org.apache.nifi.controller.StandardFlowSynchronizer.addControllerServices(StandardFlowSynchronizer.java:1374) at org.apache.nifi.controller.StandardFlowSynchronizer.addProcessGroup(StandardFlowSynchronizer.java:1316) at org.apache.nifi.controller.StandardFlowSynchronizer.addNestedProcessGroups(StandardFlowSynchronizer.java:1333) at org.apache.nifi.controller.StandardFlowSynchronizer.addProcessGroup(StandardFlowSynchronizer.java:1322) at org.apache.nifi.controller.StandardFlowSynchronizer.addNestedProcessGroups(StandardFlowSynchronizer.java:1333) at org.apache.nifi.controller.StandardFlowSynchronizer.addProcessGroup(StandardFlowSynchronizer.java:1322) at org.apache.nifi.controller.StandardFlowSynchronizer.updateFlow(StandardFlowSynchronizer.java:432) at org.apache.nifi.controller.StandardFlowSynchronizer.sync(StandardFlowSynchronizer.java:283) ... 42 common frames omitted {code}
> When importing a flow from registry, Controller Service may lose track of Referencing Components
> ------------------------------------------------------------------------------------------------
>
> Key: NIFI-7944
> URL: https://issues.apache.org/jira/browse/NIFI-7944
> Project: Apache NiFi
> Issue Type: Bug
> Components: Core Framework
> Reporter: Mark Payne
> Assignee: Mark Payne
> Priority: Major
>
> Consider a flow with a Process Group "A". Within Process Group A is a child Process Group, B. Process Group B has a Controller Service and has a Processor that references it.
> Save Process Group A into NiFi Registry.
> Import Process Group A into NiFi. Step into the Process Group and configure the child Process Group (Group B). Go to the Controller Services tab and configure the Controller Service. The service will not show the Processor as a Referencing component.
> This can result in a few different problems:
> * The Controller Service can be disabled while it's still being used.
> * If the Controller Service is enabled, along with all Referencing Components, those missing components won't show up.
> * The Controller Service can be deleted while it's still being referenced.
>
> This can lead to the following types of logs messages:
> {code:java}
> 2020-10-23 13:43:41,898 ERROR [Timer-Driven Process Thread-7] o.a.n.processors.standard.ConvertRecord ConvertRecord[id=b07355c2-8bd7-33a5-35b5-264ade413185] Failed to process StandardFlowFileRecord[uuid=824f5522-99d9-44cd-a685-eab697bdcf29,claim=StandardContentClaim [resourceClaim=StandardResourceClaim[id=1603475011829-1, container=default, section=1], offset=63, length=14],offset=0,name=824f5522-99d9-44cd-a685-eab697bdcf29,size=14]; will route to failure: org.apache.nifi.controller.service.ControllerServiceDisabledException: Cannot invoke method public abstract org.apache.nifi.serialization.record.RecordSchema org.apache.nifi.serialization.RecordSetWriterFactory.getSchema(java.util.Map,org.apache.nifi.serialization.record.RecordSchema) throws org.apache.nifi.schema.access.SchemaNotFoundException,java.io.IOException on Controller Service with identifier ca57bde0-517e-306a-ce46-10d6b8d24706 because the Controller Service's State is currently DISABLED
> org.apache.nifi.controller.service.ControllerServiceDisabledException: Cannot invoke method public abstract org.apache.nifi.serialization.record.RecordSchema org.apache.nifi.serialization.RecordSetWriterFactory.getSchema(java.util.Map,org.apache.nifi.serialization.record.RecordSchema) throws org.apache.nifi.schema.access.SchemaNotFoundException,java.io.IOException on Controller Service with identifier ca57bde0-517e-306a-ce46-10d6b8d24706 because the Controller Service's State is currently DISABLED
> at org.apache.nifi.controller.service.StandardControllerServiceInvocationHandler.invoke(StandardControllerServiceInvocationHandler.java:97)
> at com.sun.proxy.$Proxy139.getSchema(Unknown Source)
> at org.apache.nifi.processors.standard.AbstractRecordProcessor$1.process(AbstractRecordProcessor.java:149)
> at org.apache.nifi.controller.repository.StandardProcessSession.write(StandardProcessSession.java:2971)
> at org.apache.nifi.processors.standard.AbstractRecordProcessor.onTrigger(AbstractRecordProcessor.java:122)
> at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27)
> at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1173)
> at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:213)
> at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:117)
> at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2020-10-23 13:43:51,907 ERROR [Timer-Driven Process Thread-6] o.a.n.processors.standard.ConvertRecord ConvertRecord[id=b07355c2-8bd7-33a5-35b5-264ade413185] Failed to process StandardFlowFileRecord[uuid=597d9e36-3926-49d2-8f88-9200a917e29c,claim=StandardContentClaim [resourceClaim=StandardResourceClaim[id=1603475011829-1, container=default, section=1], offset=77, length=14],offset=0,name=597d9e36-3926-49d2-8f88-9200a917e29c,size=14]; will route to failure: org.apache.nifi.controller.service.ControllerServiceDisabledException: Cannot invoke method public abstract org.apache.nifi.serialization.record.RecordSchema org.apache.nifi.serialization.RecordSetWriterFactory.getSchema(java.util.Map,org.apache.nifi.serialization.record.RecordSchema) throws org.apache.nifi.schema.access.SchemaNotFoundException,java.io.IOException on Controller Service with identifier ca57bde0-517e-306a-ce46-10d6b8d24706 because the Controller Service's State is currently DISABLED
> org.apache.nifi.controller.service.ControllerServiceDisabledException: Cannot invoke method public abstract org.apache.nifi.serialization.record.RecordSchema org.apache.nifi.serialization.RecordSetWriterFactory.getSchema(java.util.Map,org.apache.nifi.serialization.record.RecordSchema) throws org.apache.nifi.schema.access.SchemaNotFoundException,java.io.IOException on Controller Service with identifier ca57bde0-517e-306a-ce46-10d6b8d24706 because the Controller Service's State is currently DISABLED
> at org.apache.nifi.controller.service.StandardControllerServiceInvocationHandler.invoke(StandardControllerServiceInvocationHandler.java:97)
> at com.sun.proxy.$Proxy139.getSchema(Unknown Source)
> at org.apache.nifi.processors.standard.AbstractRecordProcessor$1.process(AbstractRecordProcessor.java:149)
> at org.apache.nifi.controller.repository.StandardProcessSession.write(StandardProcessSession.java:2971)
> at org.apache.nifi.processors.standard.AbstractRecordProcessor.onTrigger(AbstractRecordProcessor.java:122)
> at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27)
> at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1173)
> at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:213)
> at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:117)
> at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> 2020-10-23 13:44:01,900 ERROR [Timer-Driven Process Thread-3] o.a.n.processors.standard.ConvertRecord ConvertRecord[id=b07355c2-8bd7-33a5-35b5-264ade413185] Failed to process StandardFlowFileRecord[uuid=0d71b72b-db77-4175-9deb-5f9290ea9d60,claim=StandardContentClaim [resourceClaim=StandardResourceClaim[id=1603475011829-1, container=default, section=1], offset=91, length=14],offset=0,name=0d71b72b-db77-4175-9deb-5f9290ea9d60,size=14]; will route to failure: java.lang.NullPointerException
> java.lang.NullPointerException: null
> at org.apache.nifi.processors.standard.AbstractRecordProcessor$1.process(AbstractRecordProcessor.java:149)
> at org.apache.nifi.controller.repository.StandardProcessSession.write(StandardProcessSession.java:2971)
> at org.apache.nifi.processors.standard.AbstractRecordProcessor.onTrigger(AbstractRecordProcessor.java:122)
> at org.apache.nifi.processor.AbstractProcessor.onTrigger(AbstractProcessor.java:27)
> at org.apache.nifi.controller.StandardProcessorNode.onTrigger(StandardProcessorNode.java:1173)
> at org.apache.nifi.controller.tasks.ConnectableTask.invoke(ConnectableTask.java:213)
> at org.apache.nifi.controller.scheduling.TimerDrivenSchedulingAgent$1.run(TimerDrivenSchedulingAgent.java:117)
> at org.apache.nifi.engine.FlowEngine$2.run(FlowEngine.java:110)
> at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
> at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
> at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748) {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)