You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by mc...@apache.org on 2018/03/28 16:03:35 UTC
[3/5] nifi git commit: NIFI-4895: Added backend code to give users
the ability to forcibly terminate a processor if the processor does not
complete after clicking Stop NIFI-4895: Addressed review feedback;
addressed checkstyle issues This closes #2555
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/EventDrivenSchedulingAgent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/EventDrivenSchedulingAgent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/EventDrivenSchedulingAgent.java
index 7b2e966..8b960fc 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/EventDrivenSchedulingAgent.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/EventDrivenSchedulingAgent.java
@@ -31,10 +31,13 @@ import org.apache.nifi.controller.EventBasedWorker;
import org.apache.nifi.controller.EventDrivenWorkerQueue;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ReportingTaskNode;
+import org.apache.nifi.controller.lifecycle.TaskTerminationAwareStateManager;
+import org.apache.nifi.controller.repository.ActiveProcessSessionFactory;
import org.apache.nifi.controller.repository.BatchingSessionFactory;
-import org.apache.nifi.controller.repository.ProcessContext;
+import org.apache.nifi.controller.repository.RepositoryContext;
import org.apache.nifi.controller.repository.StandardProcessSession;
import org.apache.nifi.controller.repository.StandardProcessSessionFactory;
+import org.apache.nifi.controller.repository.WeakHashMapProcessSessionFactory;
import org.apache.nifi.controller.repository.metrics.StandardFlowFileEvent;
import org.apache.nifi.controller.service.ControllerServiceProvider;
import org.apache.nifi.encrypt.StringEncryptor;
@@ -58,7 +61,7 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
private final ControllerServiceProvider serviceProvider;
private final StateManagerProvider stateManagerProvider;
private final EventDrivenWorkerQueue workerQueue;
- private final ProcessContextFactory contextFactory;
+ private final RepositoryContextFactory contextFactory;
private final AtomicInteger maxThreadCount;
private final AtomicInteger activeThreadCount = new AtomicInteger(0);
private final StringEncryptor encryptor;
@@ -66,10 +69,10 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
private volatile String adminYieldDuration = "1 sec";
private final ConcurrentMap<Connectable, AtomicLong> connectionIndexMap = new ConcurrentHashMap<>();
- private final ConcurrentMap<Connectable, ScheduleState> scheduleStates = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Connectable, LifecycleState> scheduleStates = new ConcurrentHashMap<>();
public EventDrivenSchedulingAgent(final FlowEngine flowEngine, final ControllerServiceProvider serviceProvider, final StateManagerProvider stateManagerProvider,
- final EventDrivenWorkerQueue workerQueue, final ProcessContextFactory contextFactory, final int maxThreadCount, final StringEncryptor encryptor) {
+ final EventDrivenWorkerQueue workerQueue, final RepositoryContextFactory contextFactory, final int maxThreadCount, final StringEncryptor encryptor) {
super(flowEngine);
this.serviceProvider = serviceProvider;
this.stateManagerProvider = stateManagerProvider;
@@ -98,24 +101,24 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
@Override
- public void doSchedule(final ReportingTaskNode taskNode, ScheduleState scheduleState) {
+ public void doSchedule(final ReportingTaskNode taskNode, LifecycleState scheduleState) {
throw new UnsupportedOperationException("ReportingTasks cannot be scheduled in Event-Driven Mode");
}
@Override
- public void doUnschedule(ReportingTaskNode taskNode, ScheduleState scheduleState) {
+ public void doUnschedule(ReportingTaskNode taskNode, LifecycleState scheduleState) {
throw new UnsupportedOperationException("ReportingTasks cannot be scheduled in Event-Driven Mode");
}
@Override
- public void doSchedule(final Connectable connectable, final ScheduleState scheduleState) {
+ public void doSchedule(final Connectable connectable, final LifecycleState scheduleState) {
workerQueue.resumeWork(connectable);
logger.info("Scheduled {} to run in Event-Driven mode", connectable);
scheduleStates.put(connectable, scheduleState);
}
@Override
- public void doUnschedule(final Connectable connectable, final ScheduleState scheduleState) {
+ public void doUnschedule(final Connectable connectable, final LifecycleState scheduleState) {
workerQueue.suspendWork(connectable);
logger.info("Stopped scheduling {} to run", connectable);
}
@@ -139,6 +142,10 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
@Override
+ public void incrementMaxThreadCount(int toAdd) {
+ }
+
+ @Override
public void setAdministrativeYieldDuration(final String yieldDuration) {
this.adminYieldDuration = yieldDuration;
}
@@ -171,7 +178,7 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
continue;
}
final Connectable connectable = worker.getConnectable();
- final ScheduleState scheduleState = scheduleStates.get(connectable);
+ final LifecycleState scheduleState = scheduleStates.get(connectable);
if (scheduleState == null) {
// Component not yet scheduled to run but has received events
continue;
@@ -189,26 +196,29 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
}
- final ProcessContext context = contextFactory.newProcessContext(connectable, connectionIndex);
+ final RepositoryContext context = contextFactory.newProcessContext(connectable, connectionIndex);
if (connectable instanceof ProcessorNode) {
final ProcessorNode procNode = (ProcessorNode) connectable;
- final StandardProcessContext standardProcessContext = new StandardProcessContext(procNode, serviceProvider, encryptor, getStateManager(connectable.getIdentifier()));
+ final StateManager stateManager = new TaskTerminationAwareStateManager(getStateManager(connectable.getIdentifier()), scheduleState::isTerminated);
+ final StandardProcessContext standardProcessContext = new StandardProcessContext(procNode, serviceProvider, encryptor, stateManager, scheduleState::isTerminated);
final long runNanos = procNode.getRunDuration(TimeUnit.NANOSECONDS);
final ProcessSessionFactory sessionFactory;
final StandardProcessSession rawSession;
final boolean batch;
- if (procNode.isHighThroughputSupported() && runNanos > 0L) {
- rawSession = new StandardProcessSession(context);
+ if (procNode.isSessionBatchingSupported() && runNanos > 0L) {
+ rawSession = new StandardProcessSession(context, scheduleState::isTerminated);
sessionFactory = new BatchingSessionFactory(rawSession);
batch = true;
} else {
rawSession = null;
- sessionFactory = new StandardProcessSessionFactory(context);
+ sessionFactory = new StandardProcessSessionFactory(context, scheduleState::isTerminated);
batch = false;
}
+ final ActiveProcessSessionFactory activeSessionFactory = new WeakHashMapProcessSessionFactory(sessionFactory);
+
final long startNanos = System.nanoTime();
final long finishNanos = startNanos + runNanos;
int invocationCount = 0;
@@ -216,7 +226,7 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
try {
while (shouldRun) {
- trigger(procNode, context, scheduleState, standardProcessContext, sessionFactory);
+ trigger(procNode, context, scheduleState, standardProcessContext, activeSessionFactory);
invocationCount++;
if (!batch) {
@@ -266,9 +276,10 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
onEvent(procNode);
}
} else {
- final ProcessSessionFactory sessionFactory = new StandardProcessSessionFactory(context);
+ final ProcessSessionFactory sessionFactory = new StandardProcessSessionFactory(context, scheduleState::isTerminated);
+ final ActiveProcessSessionFactory activeSessionFactory = new WeakHashMapProcessSessionFactory(sessionFactory);
final ConnectableProcessContext connectableProcessContext = new ConnectableProcessContext(connectable, encryptor, getStateManager(connectable.getIdentifier()));
- trigger(connectable, scheduleState, connectableProcessContext, sessionFactory);
+ trigger(connectable, scheduleState, connectableProcessContext, activeSessionFactory);
// See explanation above for the ProcessorNode as to why we do this.
if (Connectables.flowFilesQueued(connectable)) {
@@ -281,15 +292,15 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
}
- private void trigger(final Connectable worker, final ScheduleState scheduleState, final ConnectableProcessContext processContext, final ProcessSessionFactory sessionFactory) {
- final int newThreadCount = scheduleState.incrementActiveThreadCount();
+ private void trigger(final Connectable worker, final LifecycleState scheduleState, final ConnectableProcessContext processContext, final ActiveProcessSessionFactory sessionFactory) {
+ final int newThreadCount = scheduleState.incrementActiveThreadCount(sessionFactory);
if (newThreadCount > worker.getMaxConcurrentTasks() && worker.getMaxConcurrentTasks() > 0) {
// its possible that the worker queue could give us a worker node that is eligible to run based
// on the number of threads but another thread has already incremented the thread count, result in
// reaching the maximum number of threads. we won't know this until we atomically increment the thread count
// on the Schedule State, so we check it here. in this case, we cannot trigger the Processor, as doing so would
// result in using more than the maximum number of defined threads
- scheduleState.decrementActiveThreadCount();
+ scheduleState.decrementActiveThreadCount(sessionFactory);
return;
}
@@ -317,20 +328,20 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
}
- scheduleState.decrementActiveThreadCount();
+ scheduleState.decrementActiveThreadCount(sessionFactory);
}
}
- private void trigger(final ProcessorNode worker, final ProcessContext context, final ScheduleState scheduleState,
- final StandardProcessContext processContext, final ProcessSessionFactory sessionFactory) {
- final int newThreadCount = scheduleState.incrementActiveThreadCount();
+ private void trigger(final ProcessorNode worker, final RepositoryContext context, final LifecycleState scheduleState,
+ final StandardProcessContext processContext, final ActiveProcessSessionFactory sessionFactory) {
+ final int newThreadCount = scheduleState.incrementActiveThreadCount(sessionFactory);
if (newThreadCount > worker.getMaxConcurrentTasks() && worker.getMaxConcurrentTasks() > 0) {
// its possible that the worker queue could give us a worker node that is eligible to run based
// on the number of threads but another thread has already incremented the thread count, result in
// reaching the maximum number of threads. we won't know this until we atomically increment the thread count
// on the Schedule State, so we check it here. in this case, we cannot trigger the Processor, as doing so would
// result in using more than the maximum number of defined threads
- scheduleState.decrementActiveThreadCount();
+ scheduleState.decrementActiveThreadCount(sessionFactory);
return;
}
@@ -359,7 +370,7 @@ public class EventDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
}
- scheduleState.decrementActiveThreadCount();
+ scheduleState.decrementActiveThreadCount(sessionFactory);
}
}
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ProcessContextFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ProcessContextFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ProcessContextFactory.java
deleted file mode 100644
index 9454a4b..0000000
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/ProcessContextFactory.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.nifi.controller.scheduling;
-
-import java.util.concurrent.atomic.AtomicLong;
-
-import org.apache.nifi.connectable.Connectable;
-import org.apache.nifi.controller.repository.ContentRepository;
-import org.apache.nifi.controller.repository.CounterRepository;
-import org.apache.nifi.controller.repository.FlowFileEventRepository;
-import org.apache.nifi.controller.repository.FlowFileRepository;
-import org.apache.nifi.controller.repository.ProcessContext;
-import org.apache.nifi.provenance.ProvenanceEventRepository;
-
-public class ProcessContextFactory {
-
- private final ContentRepository contentRepo;
- private final FlowFileRepository flowFileRepo;
- private final FlowFileEventRepository flowFileEventRepo;
- private final CounterRepository counterRepo;
- private final ProvenanceEventRepository provenanceRepo;
-
- public ProcessContextFactory(final ContentRepository contentRepository, final FlowFileRepository flowFileRepository,
- final FlowFileEventRepository flowFileEventRepository, final CounterRepository counterRepository,
- final ProvenanceEventRepository provenanceRepository) {
-
- this.contentRepo = contentRepository;
- this.flowFileRepo = flowFileRepository;
- this.flowFileEventRepo = flowFileEventRepository;
- this.counterRepo = counterRepository;
- this.provenanceRepo = provenanceRepository;
- }
-
- public ProcessContext newProcessContext(final Connectable connectable, final AtomicLong connectionIndex) {
- return new ProcessContext(connectable, connectionIndex, contentRepo, flowFileRepo, flowFileEventRepo, counterRepo, provenanceRepo);
- }
-}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
index eb855d4..731314f 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/QuartzSchedulingAgent.java
@@ -16,61 +16,51 @@
*/
package org.apache.nifi.controller.scheduling;
-import java.util.ArrayList;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Callable;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.nifi.components.state.StateManager;
import org.apache.nifi.connectable.Connectable;
-import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.controller.FlowController;
-import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ReportingTaskNode;
-import org.apache.nifi.controller.tasks.ContinuallyRunConnectableTask;
-import org.apache.nifi.controller.tasks.ContinuallyRunProcessorTask;
+import org.apache.nifi.controller.tasks.ConnectableTask;
import org.apache.nifi.controller.tasks.ReportingTaskWrapper;
import org.apache.nifi.encrypt.StringEncryptor;
import org.apache.nifi.engine.FlowEngine;
-import org.apache.nifi.processor.StandardProcessContext;
import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.util.FormatUtils;
import org.quartz.CronExpression;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
public class QuartzSchedulingAgent extends AbstractSchedulingAgent {
private final Logger logger = LoggerFactory.getLogger(QuartzSchedulingAgent.class);
private final FlowController flowController;
- private final ProcessContextFactory contextFactory;
+ private final RepositoryContextFactory contextFactory;
private final StringEncryptor encryptor;
private volatile String adminYieldDuration = "1 sec";
private final Map<Object, List<AtomicBoolean>> canceledTriggers = new HashMap<>();
- public QuartzSchedulingAgent(final FlowController flowController, final FlowEngine flowEngine, final ProcessContextFactory contextFactory, final StringEncryptor enryptor) {
+ public QuartzSchedulingAgent(final FlowController flowController, final FlowEngine flowEngine, final RepositoryContextFactory contextFactory, final StringEncryptor enryptor) {
super(flowEngine);
this.flowController = flowController;
this.contextFactory = contextFactory;
this.encryptor = enryptor;
}
- private StateManager getStateManager(final String componentId) {
- return flowController.getStateManagerProvider().getStateManager(componentId);
- }
-
@Override
public void shutdown() {
}
@Override
- public void doSchedule(final ReportingTaskNode taskNode, final ScheduleState scheduleState) {
+ public void doSchedule(final ReportingTaskNode taskNode, final LifecycleState scheduleState) {
final List<AtomicBoolean> existingTriggers = canceledTriggers.get(taskNode);
if (existingTriggers != null) {
throw new IllegalStateException("Cannot schedule " + taskNode.getReportingTask().getIdentifier() + " because it is already scheduled to run");
@@ -120,7 +110,7 @@ public class QuartzSchedulingAgent extends AbstractSchedulingAgent {
}
@Override
- public synchronized void doSchedule(final Connectable connectable, final ScheduleState scheduleState) {
+ public synchronized void doSchedule(final Connectable connectable, final LifecycleState scheduleState) {
final List<AtomicBoolean> existingTriggers = canceledTriggers.get(connectable);
if (existingTriggers != null) {
throw new IllegalStateException("Cannot schedule " + connectable + " because it is already scheduled to run");
@@ -136,18 +126,7 @@ public class QuartzSchedulingAgent extends AbstractSchedulingAgent {
final List<AtomicBoolean> triggers = new ArrayList<>();
for (int i = 0; i < connectable.getMaxConcurrentTasks(); i++) {
- final Callable<Boolean> continuallyRunTask;
-
- if (connectable.getConnectableType() == ConnectableType.PROCESSOR) {
- final ProcessorNode procNode = (ProcessorNode) connectable;
-
- final StandardProcessContext standardProcContext = new StandardProcessContext(procNode, flowController, encryptor, getStateManager(connectable.getIdentifier()));
- ContinuallyRunProcessorTask runnableTask = new ContinuallyRunProcessorTask(this, procNode, flowController, contextFactory, scheduleState, standardProcContext);
- continuallyRunTask = runnableTask;
- } else {
- final ConnectableProcessContext connProcContext = new ConnectableProcessContext(connectable, encryptor, getStateManager(connectable.getIdentifier()));
- continuallyRunTask = new ContinuallyRunConnectableTask(contextFactory, connectable, scheduleState, connProcContext);
- }
+ final ConnectableTask continuallyRunTask = new ConnectableTask(this, connectable, flowController, contextFactory, scheduleState, encryptor);
final AtomicBoolean canceled = new AtomicBoolean(false);
final Runnable command = new Runnable() {
@@ -158,7 +137,7 @@ public class QuartzSchedulingAgent extends AbstractSchedulingAgent {
}
try {
- continuallyRunTask.call();
+ continuallyRunTask.invoke();
} catch (final RuntimeException re) {
throw re;
} catch (final Exception e) {
@@ -188,16 +167,16 @@ public class QuartzSchedulingAgent extends AbstractSchedulingAgent {
}
@Override
- public synchronized void doUnschedule(final Connectable connectable, final ScheduleState scheduleState) {
+ public synchronized void doUnschedule(final Connectable connectable, final LifecycleState scheduleState) {
unschedule((Object) connectable, scheduleState);
}
@Override
- public synchronized void doUnschedule(final ReportingTaskNode taskNode, final ScheduleState scheduleState) {
+ public synchronized void doUnschedule(final ReportingTaskNode taskNode, final LifecycleState scheduleState) {
unschedule((Object) taskNode, scheduleState);
}
- private void unschedule(final Object scheduled, final ScheduleState scheduleState) {
+ private void unschedule(final Object scheduled, final LifecycleState scheduleState) {
final List<AtomicBoolean> triggers = canceledTriggers.remove(scheduled);
if (triggers == null) {
throw new IllegalStateException("Cannot unschedule " + scheduled + " because it was not scheduled to run");
@@ -233,4 +212,14 @@ public class QuartzSchedulingAgent extends AbstractSchedulingAgent {
public long getAdministrativeYieldDuration(final TimeUnit timeUnit) {
return FormatUtils.getTimeDuration(adminYieldDuration, timeUnit);
}
+
+ @Override
+ public void incrementMaxThreadCount(int toAdd) {
+ final int corePoolSize = flowEngine.getCorePoolSize();
+ if (toAdd < 0 && corePoolSize + toAdd < 1) {
+ throw new IllegalStateException("Cannot remove " + (-toAdd) + " threads from pool because there are only " + corePoolSize + " threads in the pool");
+ }
+
+ flowEngine.setCorePoolSize(corePoolSize + toAdd);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/RepositoryContextFactory.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/RepositoryContextFactory.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/RepositoryContextFactory.java
new file mode 100644
index 0000000..4cffeda
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/RepositoryContextFactory.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.scheduling;
+
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.connectable.Connectable;
+import org.apache.nifi.controller.repository.ContentRepository;
+import org.apache.nifi.controller.repository.CounterRepository;
+import org.apache.nifi.controller.repository.FlowFileEventRepository;
+import org.apache.nifi.controller.repository.FlowFileRepository;
+import org.apache.nifi.controller.repository.RepositoryContext;
+import org.apache.nifi.provenance.ProvenanceEventRepository;
+
+public class RepositoryContextFactory {
+
+ private final ContentRepository contentRepo;
+ private final FlowFileRepository flowFileRepo;
+ private final FlowFileEventRepository flowFileEventRepo;
+ private final CounterRepository counterRepo;
+ private final ProvenanceEventRepository provenanceRepo;
+
+ public RepositoryContextFactory(final ContentRepository contentRepository, final FlowFileRepository flowFileRepository,
+ final FlowFileEventRepository flowFileEventRepository, final CounterRepository counterRepository,
+ final ProvenanceEventRepository provenanceRepository) {
+
+ this.contentRepo = contentRepository;
+ this.flowFileRepo = flowFileRepository;
+ this.flowFileEventRepo = flowFileEventRepository;
+ this.counterRepo = counterRepository;
+ this.provenanceRepo = provenanceRepository;
+ }
+
+ public RepositoryContext newProcessContext(final Connectable connectable, final AtomicLong connectionIndex) {
+ return new RepositoryContext(connectable, connectionIndex, contentRepo, flowFileRepo, flowFileEventRepo, counterRepo, provenanceRepo);
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
index 1155bfe..7315ef6 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/StandardProcessScheduler.java
@@ -19,6 +19,7 @@ package org.apache.nifi.controller.scheduling;
import static java.util.Objects.requireNonNull;
import java.lang.reflect.InvocationTargetException;
+import java.util.Collections;
import java.util.List;
import java.util.concurrent.Callable;
import java.util.concurrent.CompletableFuture;
@@ -38,12 +39,14 @@ import org.apache.nifi.connectable.Funnel;
import org.apache.nifi.connectable.Port;
import org.apache.nifi.controller.AbstractPort;
import org.apache.nifi.controller.ConfigurationContext;
+import org.apache.nifi.controller.FlowController;
import org.apache.nifi.controller.ProcessScheduler;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ReportingTaskNode;
import org.apache.nifi.controller.ScheduledState;
import org.apache.nifi.controller.SchedulingAgentCallback;
import org.apache.nifi.controller.StandardProcessorNode;
+import org.apache.nifi.controller.exception.ProcessorInstantiationException;
import org.apache.nifi.controller.service.ControllerServiceNode;
import org.apache.nifi.controller.service.ControllerServiceProvider;
import org.apache.nifi.encrypt.StringEncryptor;
@@ -70,11 +73,12 @@ public final class StandardProcessScheduler implements ProcessScheduler {
private static final Logger LOG = LoggerFactory.getLogger(StandardProcessScheduler.class);
private final ControllerServiceProvider controllerServiceProvider;
+ private final FlowController flowController;
private final long administrativeYieldMillis;
private final String administrativeYieldDuration;
private final StateManagerProvider stateManagerProvider;
- private final ConcurrentMap<Object, ScheduleState> scheduleStates = new ConcurrentHashMap<>();
+ private final ConcurrentMap<Object, LifecycleState> lifecycleStates = new ConcurrentHashMap<>();
private final ScheduledExecutorService frameworkTaskExecutor;
private final ConcurrentMap<SchedulingStrategy, SchedulingAgent> strategyAgentMap = new ConcurrentHashMap<>();
@@ -84,15 +88,11 @@ public final class StandardProcessScheduler implements ProcessScheduler {
private final StringEncryptor encryptor;
- public StandardProcessScheduler(
- final FlowEngine componentLifecycleThreadPool,
- final ControllerServiceProvider controllerServiceProvider,
- final StringEncryptor encryptor,
- final StateManagerProvider stateManagerProvider,
- final NiFiProperties nifiProperties
- ) {
+ public StandardProcessScheduler(final FlowEngine componentLifecycleThreadPool, final FlowController flowController, final StringEncryptor encryptor,
+ final StateManagerProvider stateManagerProvider, final NiFiProperties nifiProperties) {
this.componentLifeCycleThreadPool = componentLifecycleThreadPool;
- this.controllerServiceProvider = controllerServiceProvider;
+ this.controllerServiceProvider = flowController;
+ this.flowController = flowController;
this.encryptor = encryptor;
this.stateManagerProvider = stateManagerProvider;
@@ -170,12 +170,12 @@ public final class StandardProcessScheduler implements ProcessScheduler {
@Override
public void schedule(final ReportingTaskNode taskNode) {
- final ScheduleState scheduleState = getScheduleState(requireNonNull(taskNode));
- if (scheduleState.isScheduled()) {
+ final LifecycleState lifecycleState = getLifecycleState(requireNonNull(taskNode), true);
+ if (lifecycleState.isScheduled()) {
return;
}
- final int activeThreadCount = scheduleState.getActiveThreadCount();
+ final int activeThreadCount = lifecycleState.getActiveThreadCount();
if (activeThreadCount > 0) {
throw new IllegalStateException("Reporting Task " + taskNode.getName() + " cannot be started because it has " + activeThreadCount + " threads still running");
}
@@ -185,25 +185,25 @@ public final class StandardProcessScheduler implements ProcessScheduler {
}
final SchedulingAgent agent = getSchedulingAgent(taskNode.getSchedulingStrategy());
- scheduleState.setScheduled(true);
+ lifecycleState.setScheduled(true);
final Runnable startReportingTaskRunnable = new Runnable() {
@Override
public void run() {
- final long lastStopTime = scheduleState.getLastStopTime();
+ final long lastStopTime = lifecycleState.getLastStopTime();
final ReportingTask reportingTask = taskNode.getReportingTask();
// Continually attempt to start the Reporting Task, and if we fail sleep for a bit each time.
while (true) {
try {
- synchronized (scheduleState) {
+ synchronized (lifecycleState) {
// if no longer scheduled to run, then we're finished. This can happen, for example,
// if the @OnScheduled method throws an Exception and the user stops the reporting task
// while we're administratively yielded.
// we also check if the schedule state's last start time is equal to what it was before.
// if not, then means that the reporting task has been stopped and started again, so we should just
// bail; another thread will be responsible for invoking the @OnScheduled methods.
- if (!scheduleState.isScheduled() || scheduleState.getLastStopTime() != lastStopTime) {
+ if (!lifecycleState.isScheduled() || lifecycleState.getLastStopTime() != lastStopTime) {
return;
}
@@ -211,7 +211,7 @@ public final class StandardProcessScheduler implements ProcessScheduler {
ReflectionUtils.invokeMethodsWithAnnotation(OnScheduled.class, reportingTask, taskNode.getConfigurationContext());
}
- agent.schedule(taskNode, scheduleState);
+ agent.schedule(taskNode, lifecycleState);
return;
}
} catch (final Exception e) {
@@ -241,8 +241,8 @@ public final class StandardProcessScheduler implements ProcessScheduler {
@Override
public void unschedule(final ReportingTaskNode taskNode) {
- final ScheduleState scheduleState = getScheduleState(requireNonNull(taskNode));
- if (!scheduleState.isScheduled()) {
+ final LifecycleState lifecycleState = getLifecycleState(requireNonNull(taskNode), false);
+ if (!lifecycleState.isScheduled()) {
return;
}
@@ -256,8 +256,8 @@ public final class StandardProcessScheduler implements ProcessScheduler {
public void run() {
final ConfigurationContext configurationContext = taskNode.getConfigurationContext();
- synchronized (scheduleState) {
- scheduleState.setScheduled(false);
+ synchronized (lifecycleState) {
+ lifecycleState.setScheduled(false);
try {
try (final NarCloseable x = NarCloseable.withComponentNarLoader(reportingTask.getClass(), reportingTask.getIdentifier())) {
@@ -278,9 +278,9 @@ public final class StandardProcessScheduler implements ProcessScheduler {
}
}
- agent.unschedule(taskNode, scheduleState);
+ agent.unschedule(taskNode, lifecycleState);
- if (scheduleState.getActiveThreadCount() == 0 && scheduleState.mustCallOnStoppedMethods()) {
+ if (lifecycleState.getActiveThreadCount() == 0 && lifecycleState.mustCallOnStoppedMethods()) {
ReflectionUtils.quietlyInvokeMethodsWithAnnotation(OnStopped.class, reportingTask, configurationContext);
}
}
@@ -299,27 +299,29 @@ public final class StandardProcessScheduler implements ProcessScheduler {
*/
@Override
public synchronized CompletableFuture<Void> startProcessor(final ProcessorNode procNode, final boolean failIfStopping) {
+ final LifecycleState lifecycleState = getLifecycleState(requireNonNull(procNode), true);
+
final StandardProcessContext processContext = new StandardProcessContext(procNode, this.controllerServiceProvider,
- this.encryptor, getStateManager(procNode.getIdentifier()));
- final ScheduleState scheduleState = getScheduleState(requireNonNull(procNode));
+ this.encryptor, getStateManager(procNode.getIdentifier()), lifecycleState::isTerminated);
final CompletableFuture<Void> future = new CompletableFuture<>();
final SchedulingAgentCallback callback = new SchedulingAgentCallback() {
@Override
public void trigger() {
- getSchedulingAgent(procNode).schedule(procNode, scheduleState);
+ lifecycleState.clearTerminationFlag();
+ getSchedulingAgent(procNode).schedule(procNode, lifecycleState);
future.complete(null);
}
@Override
- public Future<?> scheduleTask(Callable<?> task) {
- scheduleState.incrementActiveThreadCount();
- return componentLifeCycleThreadPool.submit(task);
+ public Future<?> scheduleTask(final Callable<?> task) {
+ lifecycleState.incrementActiveThreadCount(null);
+ return componentMonitoringThreadPool.submit(task);
}
@Override
public void onTaskComplete() {
- scheduleState.decrementActiveThreadCount();
+ lifecycleState.decrementActiveThreadCount(null);
}
};
@@ -330,19 +332,49 @@ public final class StandardProcessScheduler implements ProcessScheduler {
/**
* Stops the given {@link Processor} by invoking its
- * {@link ProcessorNode#stop(ScheduledExecutorService, org.apache.nifi.processor.ProcessContext, SchedulingAgent, ScheduleState)}
+ * {@link ProcessorNode#stop(ScheduledExecutorService, org.apache.nifi.processor.ProcessContext, SchedulingAgent, LifecycleState)}
* method.
*
- * @see StandardProcessorNode#stop(ScheduledExecutorService, org.apache.nifi.processor.ProcessContext, SchedulingAgent, ScheduleState)
+ * @see StandardProcessorNode#stop(ScheduledExecutorService, org.apache.nifi.processor.ProcessContext, SchedulingAgent, LifecycleState)
*/
@Override
public synchronized CompletableFuture<Void> stopProcessor(final ProcessorNode procNode) {
+ final LifecycleState lifecycleState = getLifecycleState(procNode, false);
+
StandardProcessContext processContext = new StandardProcessContext(procNode, this.controllerServiceProvider,
- this.encryptor, getStateManager(procNode.getIdentifier()));
- final ScheduleState state = getScheduleState(procNode);
+ this.encryptor, getStateManager(procNode.getIdentifier()), lifecycleState::isTerminated);
LOG.info("Stopping {}", procNode);
- return procNode.stop(this, this.componentLifeCycleThreadPool, processContext, getSchedulingAgent(procNode), state);
+ return procNode.stop(this, this.componentLifeCycleThreadPool, processContext, getSchedulingAgent(procNode), lifecycleState);
+ }
+
+ @Override
+ public synchronized void terminateProcessor(final ProcessorNode procNode) {
+ if (procNode.getScheduledState() != ScheduledState.STOPPED) {
+ throw new IllegalStateException("Cannot terminate " + procNode + " because it is not currently stopped");
+ }
+
+ final LifecycleState state = getLifecycleState(procNode, false);
+ if (state.getActiveThreadCount() == 0) {
+ LOG.debug("Will not terminate {} because it has no active threads", procNode);
+ return;
+ }
+
+ LOG.debug("Terminating {}", procNode);
+
+ final int tasksTerminated = procNode.terminate();
+ state.terminate();
+
+ getSchedulingAgent(procNode).incrementMaxThreadCount(tasksTerminated);
+
+ try {
+ flowController.reload(procNode, procNode.getProcessor().getClass().getName(), procNode.getBundleCoordinate(), Collections.emptySet());
+ } catch (final ProcessorInstantiationException e) {
+ // This shouldn't happen because we already have been able to instantiate the processor before
+ LOG.error("Failed to replace instance of Processor for {} when terminating Processor", procNode);
+ }
+
+ LOG.info("Successfully terminated {} with {} active threads", procNode, tasksTerminated);
}
@Override
@@ -378,7 +410,7 @@ public final class StandardProcessScheduler implements ProcessScheduler {
@Override
public int getActiveThreadCount(final Object scheduled) {
- return getScheduleState(scheduled).getActiveThreadCount();
+ return getLifecycleState(scheduled, false).getActiveThreadCount();
}
@Override
@@ -414,22 +446,23 @@ public final class StandardProcessScheduler implements ProcessScheduler {
throw new IllegalStateException(connectable.getIdentifier() + " is disabled, so it cannot be started");
}
- final ScheduleState scheduleState = getScheduleState(requireNonNull(connectable));
- if (scheduleState.isScheduled()) {
+ final LifecycleState lifecycleState = getLifecycleState(requireNonNull(connectable), true);
+ if (lifecycleState.isScheduled()) {
return;
}
- final int activeThreads = scheduleState.getActiveThreadCount();
+ final int activeThreads = lifecycleState.getActiveThreadCount();
if (activeThreads > 0) {
throw new IllegalStateException("Port cannot be scheduled to run until its last " + activeThreads + " threads finish");
}
- getSchedulingAgent(connectable).schedule(connectable, scheduleState);
- scheduleState.setScheduled(true);
+ lifecycleState.clearTerminationFlag();
+ getSchedulingAgent(connectable).schedule(connectable, lifecycleState);
+ lifecycleState.setScheduled(true);
}
private synchronized void stopConnectable(final Connectable connectable) {
- final ScheduleState state = getScheduleState(requireNonNull(connectable));
+ final LifecycleState state = getLifecycleState(requireNonNull(connectable), false);
if (!state.isScheduled()) {
return;
}
@@ -516,25 +549,48 @@ public final class StandardProcessScheduler implements ProcessScheduler {
@Override
public boolean isScheduled(final Object scheduled) {
- final ScheduleState scheduleState = scheduleStates.get(scheduled);
- return scheduleState == null ? false : scheduleState.isScheduled();
+ final LifecycleState lifecycleState = lifecycleStates.get(scheduled);
+ return lifecycleState == null ? false : lifecycleState.isScheduled();
}
/**
- * Returns the ScheduleState that is registered for the given component; if
- * no ScheduleState current is registered, one is created and registered
+ * Returns the LifecycleState that is registered for the given component; if
+ * no LifecycleState current is registered, one is created and registered
* atomically, and then that value is returned.
*
* @param schedulable schedulable
* @return scheduled state
*/
- private ScheduleState getScheduleState(final Object schedulable) {
- ScheduleState scheduleState = this.scheduleStates.get(schedulable);
- if (scheduleState == null) {
- scheduleState = new ScheduleState();
- this.scheduleStates.putIfAbsent(schedulable, scheduleState);
+ private LifecycleState getLifecycleState(final Object schedulable, final boolean replaceTerminatedState) {
+ LifecycleState lifecycleState;
+ while (true) {
+ lifecycleState = this.lifecycleStates.get(schedulable);
+
+ if (lifecycleState == null) {
+ lifecycleState = new LifecycleState();
+ final LifecycleState existing = this.lifecycleStates.putIfAbsent(schedulable, lifecycleState);
+
+ if (existing == null) {
+ break;
+ } else {
+ continue;
+ }
+ } else if (replaceTerminatedState && lifecycleState.isTerminated()) {
+ final LifecycleState newLifecycleState = new LifecycleState();
+ final boolean replaced = this.lifecycleStates.replace(schedulable, lifecycleState, newLifecycleState);
+
+ if (replaced) {
+ lifecycleState = newLifecycleState;
+ break;
+ } else {
+ continue;
+ }
+ } else {
+ break;
+ }
}
- return scheduleState;
+
+ return lifecycleState;
}
@Override
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java
index 9dc329a..806fc67 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/scheduling/TimerDrivenSchedulingAgent.java
@@ -18,25 +18,18 @@ package org.apache.nifi.controller.scheduling;
import java.util.ArrayList;
import java.util.List;
-import java.util.concurrent.Callable;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
-import org.apache.nifi.components.state.StateManager;
import org.apache.nifi.connectable.Connectable;
-import org.apache.nifi.connectable.ConnectableType;
import org.apache.nifi.controller.FlowController;
-import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ReportingTaskNode;
-import org.apache.nifi.controller.tasks.ContinuallyRunConnectableTask;
-import org.apache.nifi.controller.tasks.ContinuallyRunProcessorTask;
+import org.apache.nifi.controller.tasks.ConnectableTask;
+import org.apache.nifi.controller.tasks.InvocationResult;
import org.apache.nifi.controller.tasks.ReportingTaskWrapper;
import org.apache.nifi.encrypt.StringEncryptor;
import org.apache.nifi.engine.FlowEngine;
-import org.apache.nifi.processor.ProcessContext;
-import org.apache.nifi.processor.StandardProcessContext;
-import org.apache.nifi.processor.exception.ProcessException;
import org.apache.nifi.util.FormatUtils;
import org.apache.nifi.util.NiFiProperties;
import org.slf4j.Logger;
@@ -48,17 +41,13 @@ public class TimerDrivenSchedulingAgent extends AbstractSchedulingAgent {
private final long noWorkYieldNanos;
private final FlowController flowController;
- private final ProcessContextFactory contextFactory;
+ private final RepositoryContextFactory contextFactory;
private final StringEncryptor encryptor;
private volatile String adminYieldDuration = "1 sec";
- public TimerDrivenSchedulingAgent(
- final FlowController flowController,
- final FlowEngine flowEngine,
- final ProcessContextFactory contextFactory,
- final StringEncryptor encryptor,
- final NiFiProperties nifiProperties) {
+ public TimerDrivenSchedulingAgent(final FlowController flowController, final FlowEngine flowEngine, final RepositoryContextFactory contextFactory,
+ final StringEncryptor encryptor, final NiFiProperties nifiProperties) {
super(flowEngine);
this.flowController = flowController;
this.contextFactory = contextFactory;
@@ -72,17 +61,13 @@ public class TimerDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
}
- private StateManager getStateManager(final String componentId) {
- return flowController.getStateManagerProvider().getStateManager(componentId);
- }
-
@Override
public void shutdown() {
flowEngine.shutdown();
}
@Override
- public void doSchedule(final ReportingTaskNode taskNode, final ScheduleState scheduleState) {
+ public void doSchedule(final ReportingTaskNode taskNode, final LifecycleState scheduleState) {
final Runnable reportingTaskWrapper = new ReportingTaskWrapper(taskNode, scheduleState);
final long schedulingNanos = taskNode.getSchedulingPeriod(TimeUnit.NANOSECONDS);
@@ -95,100 +80,19 @@ public class TimerDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
@Override
- public void doSchedule(final Connectable connectable, final ScheduleState scheduleState) {
-
+ public void doSchedule(final Connectable connectable, final LifecycleState scheduleState) {
final List<ScheduledFuture<?>> futures = new ArrayList<>();
- for (int i = 0; i < connectable.getMaxConcurrentTasks(); i++) {
- final Callable<Boolean> continuallyRunTask;
- final ProcessContext processContext;
+ final ConnectableTask connectableTask = new ConnectableTask(this, connectable, flowController, contextFactory, scheduleState, encryptor);
+ for (int i = 0; i < connectable.getMaxConcurrentTasks(); i++) {
// Determine the task to run and create it.
- if (connectable.getConnectableType() == ConnectableType.PROCESSOR) {
- final ProcessorNode procNode = (ProcessorNode) connectable;
- final StandardProcessContext standardProcContext = new StandardProcessContext(procNode, flowController, encryptor, getStateManager(connectable.getIdentifier()));
- final ContinuallyRunProcessorTask runnableTask = new ContinuallyRunProcessorTask(this, procNode, flowController,
- contextFactory, scheduleState, standardProcContext);
-
- continuallyRunTask = runnableTask;
- processContext = standardProcContext;
- } else {
- processContext = new ConnectableProcessContext(connectable, encryptor, getStateManager(connectable.getIdentifier()));
- continuallyRunTask = new ContinuallyRunConnectableTask(contextFactory, connectable, scheduleState, processContext);
- }
-
final AtomicReference<ScheduledFuture<?>> futureRef = new AtomicReference<>();
- final Runnable yieldDetectionRunnable = new Runnable() {
- @Override
- public void run() {
- // Call the continually run task. It will return a boolean indicating whether or not we should yield
- // based on a lack of work for to do for the component.
- final boolean shouldYield;
- try {
- shouldYield = continuallyRunTask.call();
- } catch (final RuntimeException re) {
- throw re;
- } catch (final Exception e) {
- throw new ProcessException(e);
- }
-
- // If the component is yielded, cancel its future and re-submit it to run again
- // after the yield has expired.
- final long newYieldExpiration = connectable.getYieldExpiration();
- final long now = System.currentTimeMillis();
- if (newYieldExpiration > now) {
- final long yieldMillis = newYieldExpiration - now;
- final long scheduleMillis = connectable.getSchedulingPeriod(TimeUnit.MILLISECONDS);
- final ScheduledFuture<?> scheduledFuture = futureRef.get();
- if (scheduledFuture == null) {
- return;
- }
-
- // If we are able to cancel the future, create a new one and update the ScheduleState so that it has
- // an accurate accounting of which futures are outstanding; we must then also update the futureRef
- // so that we can do this again the next time that the component is yielded.
- if (scheduledFuture.cancel(false)) {
- final long yieldNanos = Math.max(TimeUnit.MILLISECONDS.toNanos(scheduleMillis), TimeUnit.MILLISECONDS.toNanos(yieldMillis));
-
- synchronized (scheduleState) {
- if (scheduleState.isScheduled()) {
- final ScheduledFuture<?> newFuture = flowEngine.scheduleWithFixedDelay(this, yieldNanos,
- connectable.getSchedulingPeriod(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
-
- scheduleState.replaceFuture(scheduledFuture, newFuture);
- futureRef.set(newFuture);
- }
- }
- }
- } else if (noWorkYieldNanos > 0L && shouldYield) {
- // Component itself didn't yield but there was no work to do, so the framework will choose
- // to yield the component automatically for a short period of time.
- final ScheduledFuture<?> scheduledFuture = futureRef.get();
- if (scheduledFuture == null) {
- return;
- }
-
- // If we are able to cancel the future, create a new one and update the ScheduleState so that it has
- // an accurate accounting of which futures are outstanding; we must then also update the futureRef
- // so that we can do this again the next time that the component is yielded.
- if (scheduledFuture.cancel(false)) {
- synchronized (scheduleState) {
- if (scheduleState.isScheduled()) {
- final ScheduledFuture<?> newFuture = flowEngine.scheduleWithFixedDelay(this, noWorkYieldNanos,
- connectable.getSchedulingPeriod(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
-
- scheduleState.replaceFuture(scheduledFuture, newFuture);
- futureRef.set(newFuture);
- }
- }
- }
- }
- }
- };
+ final Runnable trigger = createTrigger(connectableTask, scheduleState, futureRef);
// Schedule the task to run
- final ScheduledFuture<?> future = flowEngine.scheduleWithFixedDelay(yieldDetectionRunnable, 0L,
- connectable.getSchedulingPeriod(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
+ final ScheduledFuture<?> future = flowEngine.scheduleWithFixedDelay(trigger, 0L,
+ connectable.getSchedulingPeriod(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
// now that we have the future, set the atomic reference so that if the component is yielded we
// are able to then cancel this future.
@@ -202,8 +106,78 @@ public class TimerDrivenSchedulingAgent extends AbstractSchedulingAgent {
logger.info("Scheduled {} to run with {} threads", connectable, connectable.getMaxConcurrentTasks());
}
+
+ private Runnable createTrigger(final ConnectableTask connectableTask, final LifecycleState scheduleState, final AtomicReference<ScheduledFuture<?>> futureRef) {
+ final Connectable connectable = connectableTask.getConnectable();
+ final Runnable yieldDetectionRunnable = new Runnable() {
+ @Override
+ public void run() {
+ // Call the task. It will return a boolean indicating whether or not we should yield
+ // based on a lack of work for to do for the component.
+ final InvocationResult invocationResult = connectableTask.invoke();
+ if (invocationResult.isYield()) {
+ logger.debug("Yielding {} due to {}", connectable, invocationResult.getYieldExplanation());
+ }
+
+ // If the component is yielded, cancel its future and re-submit it to run again
+ // after the yield has expired.
+ final long newYieldExpiration = connectable.getYieldExpiration();
+ final long now = System.currentTimeMillis();
+ if (newYieldExpiration > now) {
+ final long yieldMillis = newYieldExpiration - now;
+ final long scheduleMillis = connectable.getSchedulingPeriod(TimeUnit.MILLISECONDS);
+ final ScheduledFuture<?> scheduledFuture = futureRef.get();
+ if (scheduledFuture == null) {
+ return;
+ }
+
+ // If we are able to cancel the future, create a new one and update the ScheduleState so that it has
+ // an accurate accounting of which futures are outstanding; we must then also update the futureRef
+ // so that we can do this again the next time that the component is yielded.
+ if (scheduledFuture.cancel(false)) {
+ final long yieldNanos = Math.max(TimeUnit.MILLISECONDS.toNanos(scheduleMillis), TimeUnit.MILLISECONDS.toNanos(yieldMillis));
+
+ synchronized (scheduleState) {
+ if (scheduleState.isScheduled()) {
+ final long schedulingNanos = connectable.getSchedulingPeriod(TimeUnit.NANOSECONDS);
+ final ScheduledFuture<?> newFuture = flowEngine.scheduleWithFixedDelay(this, yieldNanos, schedulingNanos, TimeUnit.NANOSECONDS);
+
+ scheduleState.replaceFuture(scheduledFuture, newFuture);
+ futureRef.set(newFuture);
+ }
+ }
+ }
+ } else if (noWorkYieldNanos > 0L && invocationResult.isYield()) {
+ // Component itself didn't yield but there was no work to do, so the framework will choose
+ // to yield the component automatically for a short period of time.
+ final ScheduledFuture<?> scheduledFuture = futureRef.get();
+ if (scheduledFuture == null) {
+ return;
+ }
+
+ // If we are able to cancel the future, create a new one and update the ScheduleState so that it has
+ // an accurate accounting of which futures are outstanding; we must then also update the futureRef
+ // so that we can do this again the next time that the component is yielded.
+ if (scheduledFuture.cancel(false)) {
+ synchronized (scheduleState) {
+ if (scheduleState.isScheduled()) {
+ final ScheduledFuture<?> newFuture = flowEngine.scheduleWithFixedDelay(this, noWorkYieldNanos,
+ connectable.getSchedulingPeriod(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
+
+ scheduleState.replaceFuture(scheduledFuture, newFuture);
+ futureRef.set(newFuture);
+ }
+ }
+ }
+ }
+ }
+ };
+
+ return yieldDetectionRunnable;
+ }
+
@Override
- public void doUnschedule(final Connectable connectable, final ScheduleState scheduleState) {
+ public void doUnschedule(final Connectable connectable, final LifecycleState scheduleState) {
for (final ScheduledFuture<?> future : scheduleState.getFutures()) {
// stop scheduling to run but do not interrupt currently running tasks.
future.cancel(false);
@@ -213,7 +187,7 @@ public class TimerDrivenSchedulingAgent extends AbstractSchedulingAgent {
}
@Override
- public void doUnschedule(final ReportingTaskNode taskNode, final ScheduleState scheduleState) {
+ public void doUnschedule(final ReportingTaskNode taskNode, final LifecycleState scheduleState) {
for (final ScheduledFuture<?> future : scheduleState.getFutures()) {
// stop scheduling to run but do not interrupt currently running tasks.
future.cancel(false);
@@ -245,4 +219,13 @@ public class TimerDrivenSchedulingAgent extends AbstractSchedulingAgent {
public void setMaxThreadCount(final int maxThreads) {
}
+ @Override
+ public void incrementMaxThreadCount(int toAdd) {
+ final int corePoolSize = flowEngine.getCorePoolSize();
+ if (toAdd < 0 && corePoolSize + toAdd < 1) {
+ throw new IllegalStateException("Cannot remove " + (-toAdd) + " threads from pool because there are only " + corePoolSize + " threads in the pool");
+ }
+
+ flowEngine.setCorePoolSize(corePoolSize + toAdd);
+ }
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceDetails.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceDetails.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceDetails.java
index 3db77c9..b021369 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceDetails.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/ControllerServiceDetails.java
@@ -19,7 +19,7 @@ package org.apache.nifi.controller.service;
import org.apache.nifi.bundle.BundleCoordinate;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.LoggableComponent;
-import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.controller.TerminationAwareLogger;
/**
* Holder for StandardControllerServiceNode to atomically swap out the component.
@@ -28,7 +28,7 @@ public class ControllerServiceDetails {
private final ControllerService proxiedControllerService;
private final ControllerService implementation;
- private final ComponentLog componentLog;
+ private final TerminationAwareLogger componentLog;
private final BundleCoordinate bundleCoordinate;
private final ControllerServiceInvocationHandler invocationHandler;
@@ -50,7 +50,7 @@ public class ControllerServiceDetails {
return implementation;
}
- public ComponentLog getComponentLog() {
+ public TerminationAwareLogger getComponentLog() {
return componentLog;
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
index a7293d7..9d26eef 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceNode.java
@@ -54,6 +54,7 @@ import org.apache.nifi.controller.ConfiguredComponent;
import org.apache.nifi.controller.ControllerService;
import org.apache.nifi.controller.LoggableComponent;
import org.apache.nifi.controller.ReloadComponent;
+import org.apache.nifi.controller.TerminationAwareLogger;
import org.apache.nifi.controller.ValidationContextFactory;
import org.apache.nifi.controller.exception.ControllerServiceInstantiationException;
import org.apache.nifi.groups.ProcessGroup;
@@ -112,7 +113,7 @@ public class StandardControllerServiceNode extends AbstractConfiguredComponent i
}
@Override
- public ComponentLog getLogger() {
+ public TerminationAwareLogger getLogger() {
return controllerServiceHolder.get().getComponentLog();
}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
index 271cf08..cba1978 100644
--- a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/service/StandardControllerServiceProvider.java
@@ -54,6 +54,7 @@ import org.apache.nifi.controller.LoggableComponent;
import org.apache.nifi.controller.ProcessorNode;
import org.apache.nifi.controller.ReportingTaskNode;
import org.apache.nifi.controller.ScheduledState;
+import org.apache.nifi.controller.TerminationAwareLogger;
import org.apache.nifi.controller.ValidationContextFactory;
import org.apache.nifi.controller.exception.ComponentLifeCycleException;
import org.apache.nifi.controller.exception.ControllerServiceInstantiationException;
@@ -147,12 +148,14 @@ public class StandardControllerServiceProvider implements ControllerServiceProvi
logger.info("Created Controller Service of type {} with identifier {}", type, id);
final ComponentLog serviceLogger = new SimpleProcessLogger(id, originalService);
- originalService.initialize(new StandardControllerServiceInitializationContext(id, serviceLogger, this, getStateManager(id), nifiProperties));
+ final TerminationAwareLogger terminationAwareLogger = new TerminationAwareLogger(serviceLogger);
+
+ originalService.initialize(new StandardControllerServiceInitializationContext(id, terminationAwareLogger, this, getStateManager(id), nifiProperties));
final ValidationContextFactory validationContextFactory = new StandardValidationContextFactory(this, variableRegistry);
- final LoggableComponent<ControllerService> originalLoggableComponent = new LoggableComponent<>(originalService, bundleCoordinate, serviceLogger);
- final LoggableComponent<ControllerService> proxiedLoggableComponent = new LoggableComponent<>(proxiedService, bundleCoordinate, serviceLogger);
+ final LoggableComponent<ControllerService> originalLoggableComponent = new LoggableComponent<>(originalService, bundleCoordinate, terminationAwareLogger);
+ final LoggableComponent<ControllerService> proxiedLoggableComponent = new LoggableComponent<>(proxiedService, bundleCoordinate, terminationAwareLogger);
final ComponentVariableRegistry componentVarRegistry = new StandardComponentVariableRegistry(this.variableRegistry);
final ControllerServiceNode serviceNode = new StandardControllerServiceNode(originalLoggableComponent, proxiedLoggableComponent, invocationHandler,
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ActiveTask.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ActiveTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ActiveTask.java
new file mode 100644
index 0000000..614995b
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ActiveTask.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.controller.tasks;
+
+public class ActiveTask {
+ private final long startTime;
+ private volatile boolean terminated;
+
+ public ActiveTask(final long startTime) {
+ this.startTime = startTime;
+ }
+
+ public long getStartTime() {
+ return startTime;
+ }
+
+ public boolean isTerminated() {
+ return terminated;
+ }
+
+ public void terminate() {
+ this.terminated = true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/nifi/blob/f394c874/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java
new file mode 100644
index 0000000..b0ba446
--- /dev/null
+++ b/nifi-nar-bundles/nifi-framework-bundle/nifi-framework/nifi-framework-core/src/main/java/org/apache/nifi/controller/tasks/ConnectableTask.java
@@ -0,0 +1,259 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.nifi.controller.tasks;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+
+import org.apache.nifi.components.state.StateManager;
+import org.apache.nifi.connectable.Connectable;
+import org.apache.nifi.controller.FlowController;
+import org.apache.nifi.controller.ProcessorNode;
+import org.apache.nifi.controller.ScheduledState;
+import org.apache.nifi.controller.lifecycle.TaskTerminationAwareStateManager;
+import org.apache.nifi.controller.repository.ActiveProcessSessionFactory;
+import org.apache.nifi.controller.repository.BatchingSessionFactory;
+import org.apache.nifi.controller.repository.RepositoryContext;
+import org.apache.nifi.controller.repository.StandardProcessSession;
+import org.apache.nifi.controller.repository.StandardProcessSessionFactory;
+import org.apache.nifi.controller.repository.WeakHashMapProcessSessionFactory;
+import org.apache.nifi.controller.repository.metrics.StandardFlowFileEvent;
+import org.apache.nifi.controller.scheduling.ConnectableProcessContext;
+import org.apache.nifi.controller.scheduling.RepositoryContextFactory;
+import org.apache.nifi.controller.scheduling.LifecycleState;
+import org.apache.nifi.controller.scheduling.SchedulingAgent;
+import org.apache.nifi.encrypt.StringEncryptor;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.nar.NarCloseable;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSessionFactory;
+import org.apache.nifi.processor.SimpleProcessLogger;
+import org.apache.nifi.processor.StandardProcessContext;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.exception.TerminatedTaskException;
+import org.apache.nifi.util.Connectables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Continually runs a processor as long as the processor has work to do. {@link #call()} will return <code>true</code> if the processor should be yielded, <code>false</code> otherwise.
+ */
+public class ConnectableTask {
+
+ private static final Logger logger = LoggerFactory.getLogger(ConnectableTask.class);
+
+ private final SchedulingAgent schedulingAgent;
+ private final Connectable connectable;
+ private final RepositoryContext repositoryContext;
+ private final LifecycleState scheduleState;
+ private final ProcessContext processContext;
+ private final FlowController flowController;
+ private final int numRelationships;
+ private final boolean hasNonLoopConnection;
+
+
+ public ConnectableTask(final SchedulingAgent schedulingAgent, final Connectable connectable,
+ final FlowController flowController, final RepositoryContextFactory contextFactory, final LifecycleState scheduleState,
+ final StringEncryptor encryptor) {
+
+ this.schedulingAgent = schedulingAgent;
+ this.connectable = connectable;
+ this.scheduleState = scheduleState;
+ this.numRelationships = connectable.getRelationships().size();
+ this.flowController = flowController;
+ this.hasNonLoopConnection = Connectables.hasNonLoopConnection(connectable);
+
+ final StateManager stateManager = new TaskTerminationAwareStateManager(flowController.getStateManagerProvider().getStateManager(connectable.getIdentifier()), scheduleState::isTerminated);
+ if (connectable instanceof ProcessorNode) {
+ processContext = new StandardProcessContext((ProcessorNode) connectable, flowController, encryptor, stateManager, scheduleState::isTerminated);
+ } else {
+ processContext = new ConnectableProcessContext(connectable, encryptor, stateManager);
+ }
+
+ repositoryContext = contextFactory.newProcessContext(connectable, new AtomicLong(0L));
+ }
+
+ public Connectable getConnectable() {
+ return connectable;
+ }
+
+ private boolean isRunOnCluster(final FlowController flowController) {
+ return !connectable.isIsolated() || !flowController.isConfiguredForClustering() || flowController.isPrimary();
+ }
+
+ private boolean isYielded() {
+ // after one yield period, the scheduling agent could call this again when
+ // yieldExpiration == currentTime, and we don't want that to still be considered 'yielded'
+ // so this uses ">" instead of ">="
+ return connectable.getYieldExpiration() > System.currentTimeMillis();
+ }
+
+ private boolean isWorkToDo() {
+ return connectable.isTriggerWhenEmpty() || !connectable.hasIncomingConnection() || !hasNonLoopConnection || Connectables.flowFilesQueued(connectable);
+ }
+
+ private boolean isBackPressureEngaged() {
+ return connectable.getIncomingConnections().stream()
+ .filter(con -> con.getSource() == connectable)
+ .map(con -> con.getFlowFileQueue())
+ .anyMatch(queue -> queue.isFull());
+ }
+
+ public InvocationResult invoke() {
+ if (scheduleState.isTerminated()) {
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+ // make sure processor is not yielded
+ if (isYielded()) {
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+ // make sure that either we're not clustered or this processor runs on all nodes or that this is the primary node
+ if (!isRunOnCluster(flowController)) {
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+ // Make sure processor has work to do. This means that it meets one of these criteria:
+ // * It is annotated with @TriggerWhenEmpty
+ // * It has data in an incoming Connection
+ // * It has no incoming connections
+ // * All incoming connections are self-loops
+ if (!isWorkToDo()) {
+ return InvocationResult.yield("No work to do");
+ }
+
+ if (numRelationships > 0) {
+ final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
+ if (!repositoryContext.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships)) {
+ return InvocationResult.yield("Backpressure Applied");
+ }
+ }
+
+ final long batchNanos = connectable.getRunDuration(TimeUnit.NANOSECONDS);
+ final ProcessSessionFactory sessionFactory;
+ final StandardProcessSession rawSession;
+ final boolean batch;
+ if (connectable.isSessionBatchingSupported() && batchNanos > 0L) {
+ rawSession = new StandardProcessSession(repositoryContext, scheduleState::isTerminated);
+ sessionFactory = new BatchingSessionFactory(rawSession);
+ batch = true;
+ } else {
+ rawSession = null;
+ sessionFactory = new StandardProcessSessionFactory(repositoryContext, scheduleState::isTerminated);
+ batch = false;
+ }
+
+ final ActiveProcessSessionFactory activeSessionFactory = new WeakHashMapProcessSessionFactory(sessionFactory);
+ scheduleState.incrementActiveThreadCount(activeSessionFactory);
+
+ final long startNanos = System.nanoTime();
+ final long finishIfBackpressureEngaged = startNanos + (batchNanos / 25L);
+ final long finishNanos = startNanos + batchNanos;
+ int invocationCount = 0;
+
+ final String originalThreadName = Thread.currentThread().getName();
+ try {
+ try (final AutoCloseable ncl = NarCloseable.withComponentNarLoader(connectable.getRunnableComponent().getClass(), connectable.getIdentifier())) {
+ boolean shouldRun = connectable.getScheduledState() == ScheduledState.RUNNING;
+ while (shouldRun) {
+ connectable.onTrigger(processContext, activeSessionFactory);
+ invocationCount++;
+
+ if (!batch) {
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+ final long nanoTime = System.nanoTime();
+ if (nanoTime > finishNanos) {
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+ if (nanoTime > finishIfBackpressureEngaged && isBackPressureEngaged()) {
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+ if (connectable.getScheduledState() != ScheduledState.RUNNING) {
+ break;
+ }
+
+ if (!isWorkToDo()) {
+ break;
+ }
+ if (isYielded()) {
+ break;
+ }
+
+ if (numRelationships > 0) {
+ final int requiredNumberOfAvailableRelationships = connectable.isTriggerWhenAnyDestinationAvailable() ? 1 : numRelationships;
+ shouldRun = repositoryContext.isRelationshipAvailabilitySatisfied(requiredNumberOfAvailableRelationships);
+ }
+ }
+ } catch (final TerminatedTaskException tte) {
+ final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
+ procLog.info("Failed to process session due to task being terminated", new Object[] {tte});
+ } catch (final ProcessException pe) {
+ final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
+ procLog.error("Failed to process session due to {}", new Object[] {pe});
+ } catch (final Throwable t) {
+ // Use ComponentLog to log the event so that a bulletin will be created for this processor
+ final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
+ procLog.error("{} failed to process session due to {}; Processor Administratively Yielded for {}",
+ new Object[] {connectable.getRunnableComponent(), t, schedulingAgent.getAdministrativeYieldDuration()}, t);
+ logger.warn("Administratively Yielding {} due to uncaught Exception: {}", connectable.getRunnableComponent(), t.toString(), t);
+
+ connectable.yield(schedulingAgent.getAdministrativeYieldDuration(TimeUnit.NANOSECONDS), TimeUnit.NANOSECONDS);
+ }
+ } finally {
+ try {
+ if (batch) {
+ try {
+ rawSession.commit();
+ } catch (final Exception e) {
+ final ComponentLog procLog = new SimpleProcessLogger(connectable.getIdentifier(), connectable.getRunnableComponent());
+ procLog.error("Failed to commit session {} due to {}; rolling back", new Object[] { rawSession, e.toString() }, e);
+
+ try {
+ rawSession.rollback(true);
+ } catch (final Exception e1) {
+ procLog.error("Failed to roll back session {} due to {}", new Object[] { rawSession, e.toString() }, e);
+ }
+ }
+ }
+
+ final long processingNanos = System.nanoTime() - startNanos;
+
+ try {
+ final StandardFlowFileEvent procEvent = new StandardFlowFileEvent(connectable.getIdentifier());
+ procEvent.setProcessingNanos(processingNanos);
+ procEvent.setInvocations(invocationCount);
+ repositoryContext.getFlowFileEventRepository().updateRepository(procEvent);
+ } catch (final IOException e) {
+ logger.error("Unable to update FlowFileEvent Repository for {}; statistics may be inaccurate. Reason for failure: {}", connectable.getRunnableComponent(), e.toString());
+ logger.error("", e);
+ }
+ } finally {
+ scheduleState.decrementActiveThreadCount(activeSessionFactory);
+ Thread.currentThread().setName(originalThreadName);
+ }
+ }
+
+ return InvocationResult.DO_NOT_YIELD;
+ }
+
+}