You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ka...@apache.org on 2011/10/06 22:14:29 UTC
svn commit: r1179815 - in /incubator/oozie/trunk:
core/src/main/java/org/apache/oozie/service/CoordMaterializeTriggerService.java
core/src/main/resources/oozie-default.xml release-log.txt
Author: kamrul
Date: Thu Oct 6 20:14:29 2011
New Revision: 1179815
URL: http://svn.apache.org/viewvc?rev=1179815&view=rev
Log:
Closes OOZIE-570 Oozie bundle is running but not materializing new actions
Modified:
incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/CoordMaterializeTriggerService.java
incubator/oozie/trunk/core/src/main/resources/oozie-default.xml
incubator/oozie/trunk/release-log.txt
Modified: incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/CoordMaterializeTriggerService.java
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/CoordMaterializeTriggerService.java?rev=1179815&r1=1179814&r2=1179815&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/CoordMaterializeTriggerService.java (original)
+++ incubator/oozie/trunk/core/src/main/java/org/apache/oozie/service/CoordMaterializeTriggerService.java Thu Oct 6 20:14:29 2011
@@ -6,9 +6,9 @@
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
- *
+ *
* http://www.apache.org/licenses/LICENSE-2.0
- *
+ *
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -51,11 +51,16 @@ public class CoordMaterializeTriggerServ
* The number of callables to be queued in a batch.
*/
public static final String CONF_CALLABLE_BATCH_SIZE = CONF_PREFIX + "callable.batch.size";
+ /**
+ * The number of coordinator jobs to be picked for materialization at a given time.
+ */
+ public static final String CONF_MATERIALIZATION_SYSTEM_LIMIT = CONF_PREFIX + "materialization.system.limit";
private static final String INSTRUMENTATION_GROUP = "coord_job_mat";
private static final String INSTR_MAT_JOBS_COUNTER = "jobs";
private static final int CONF_LOOKUP_INTERVAL_DEFAULT = 300;
private static final int CONF_MATERIALIZATION_WINDOW_DEFAULT = 3600;
+ private static final int CONF_MATERIALIZATION_SYSTEM_LIMIT_DEFAULT = 50;
/**
* This runnable class will run in every "interval" to queue CoordMaterializeTransitionXCommand.
@@ -109,26 +114,29 @@ public class CoordMaterializeTriggerServ
// get current date
Date currDate = new Date(new Date().getTime() + CONF_LOOKUP_INTERVAL_DEFAULT * 1000);
// get list of all jobs that have actions that should be materialized.
- CoordJobsToBeMaterializedJPAExecutor cmatcmd = new CoordJobsToBeMaterializedJPAExecutor(currDate, 50);
+ int materializationLimit = Services.get().getConf()
+ .getInt(CONF_MATERIALIZATION_SYSTEM_LIMIT, CONF_MATERIALIZATION_SYSTEM_LIMIT_DEFAULT);
+ CoordJobsToBeMaterializedJPAExecutor cmatcmd = new CoordJobsToBeMaterializedJPAExecutor(currDate,
+ materializationLimit);
List<CoordinatorJobBean> materializeJobs = jpaService.execute(cmatcmd);
LOG.debug("CoordMaterializeTriggerService - Curr Date= " + currDate + ", Num jobs to materialize = "
+ materializeJobs.size());
for (CoordinatorJobBean coordJob : materializeJobs) {
- Services.get().get(InstrumentationService.class).get().incr(INSTRUMENTATION_GROUP,
- INSTR_MAT_JOBS_COUNTER, 1);
+ Services.get().get(InstrumentationService.class).get()
+ .incr(INSTRUMENTATION_GROUP, INSTR_MAT_JOBS_COUNTER, 1);
int numWaitingActions = jpaService
.execute(new CoordActionsActiveCountJPAExecutor(coordJob.getId()));
- LOG.debug("Job :" + coordJob.getId() + " numWaitingActions : " + numWaitingActions + " MatThrottle : "
- + coordJob.getMatThrottling());
+ LOG.debug("Job :" + coordJob.getId() + " numWaitingActions : " + numWaitingActions
+ + " MatThrottle : " + coordJob.getMatThrottling());
+ // update lastModifiedTime so next time others might have higher chance to get pick up
+ coordJob.setLastModifiedTime(new Date());
+ jpaService.execute(new CoordJobUpdateJPAExecutor(coordJob));
if (numWaitingActions >= coordJob.getMatThrottling()) {
LOG.debug("Materialization skipped for JobID [" + coordJob.getId() + " already waiting "
+ numWaitingActions + " actions. MatThrottle is : " + coordJob.getMatThrottling());
continue;
}
queueCallable(new CoordMaterializeTransitionXCommand(coordJob.getId(), materializationWindow));
- //update lastModifiedTime so next time others might have higher chance to get pick up
- coordJob.setLastModifiedTime(new Date());
- jpaService.execute(new CoordJobUpdateJPAExecutor(coordJob));
}
Modified: incubator/oozie/trunk/core/src/main/resources/oozie-default.xml
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/core/src/main/resources/oozie-default.xml?rev=1179815&r1=1179814&r2=1179815&view=diff
==============================================================================
--- incubator/oozie/trunk/core/src/main/resources/oozie-default.xml (original)
+++ incubator/oozie/trunk/core/src/main/resources/oozie-default.xml Thu Oct 6 20:14:29 2011
@@ -299,6 +299,14 @@
</property>
<property>
+ <name>oozie.service.CoordMaterializeTriggerService.materialization.system.limit</name>
+ <value>50</value>
+ <description>
+ This value determines the number of coordinator jobs to be materialized at a given time.
+ </description>
+ </property>
+
+ <property>
<name>oozie.service.coord.normal.default.timeout
</name>
<value>10080</value>
Modified: incubator/oozie/trunk/release-log.txt
URL: http://svn.apache.org/viewvc/incubator/oozie/trunk/release-log.txt?rev=1179815&r1=1179814&r2=1179815&view=diff
==============================================================================
--- incubator/oozie/trunk/release-log.txt (original)
+++ incubator/oozie/trunk/release-log.txt Thu Oct 6 20:14:29 2011
@@ -1,4 +1,5 @@
-- Oozie 3.1.1 release
+OOZIE-570 Oozie bundle is running but not materializing new actions
OOZIE-25 Removing confusing exception trace during wf suspend/kill/resume
OOZIE-567 Is sub-workflow lib directory not used? Libraries of both parent and child workflows need to be added to classpath
OOZIE-564 Embedded pig within python fails when run using oozie